2024-09-14T08:22:14.482 INFO:root:teuthology version: 1.2.2.dev2+g3752d38 2024-09-14T08:22:14.504 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-14T08:22:14.591 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2024-09-10_20:16:17-rbd-main-distro-default-smithi/7900492 branch: main description: rbd/mirror-thrash/{base/install clients/mirror cluster/{2-node openstack} conf/{disable-pool-app} msgr-failures/few objectstore/bluestore-comp-zlib policy/simple rbd-mirror/four-per-cluster supported-random-distro$/{centos_latest} workloads/rbd-mirror-snapshot-stress-workunit-minimum} email: ceph-qa@ceph.com first_in_suite: false job_id: '7900492' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi meta: - desc: run two ceph clusters and install rbd-mirror - desc: configure the permissions for client.mirror - desc: 2 ceph clusters with 1 mon and 3 osds each - desc: run four rbd-mirror daemons per cluster - desc: run the rbd_mirror_stress.sh workunit to test the rbd-mirror daemon name: teuthology-2024-09-10_20:16:17-rbd-main-distro-default-smithi no_nested_subset: false openstack: - volumes: count: 3 size: 30 os_type: centos os_version: 9.stream overrides: admin_socket: branch: main ceph: conf: client: debug rbd: 20 debug rbd_mirror: 15 log to stderr: false rbd default features: 125 rbd mirror image policy type: simple client.mirror.0: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid client.mirror.1: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid client.mirror.2: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid client.mirror.3: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid global: mon client directed command retry: 5 mon warn on pool no app: false ms inject socket failures: 5000 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: bluestore block size: 96636764160 bluestore compression algorithm: zlib bluestore compression mode: aggressive bluestore fsck on mount: true debug bluefs: 1/20 debug bluestore: 1/20 debug ms: 1 debug osd: 20 debug rocksdb: 4/10 mon osd backfillfull_ratio: 0.85 mon osd full ratio: 0.9 mon osd nearfull ratio: 0.8 osd failsafe full ratio: 0.95 osd objectstore: bluestore flavor: default fs: xfs log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) - \(OSD_SLOW_PING_TIME sha1: 7f24e589328ae9e639c0009d93340e5b235d0a47 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: 7f24e589328ae9e639c0009d93340e5b235d0a47 thrashosds: bdev_inject_crash: 2 bdev_inject_crash_probability: 0.5 workunit: branch: main sha1: 0d61375b8ab53be4c233f5b93e463edfdff9316a owner: scheduled_teuthology@teuthology priority: 950 repo: https://git.ceph.com/ceph.git roles: - - cluster1.mon.a - cluster1.mgr.x - cluster2.mgr.x - cluster1.osd.0 - cluster1.osd.1 - cluster1.osd.2 - cluster1.client.0 - cluster2.client.0 - - cluster2.mon.a - cluster2.osd.0 - cluster2.osd.1 - cluster2.osd.2 - cluster1.client.mirror - cluster1.client.mirror.0 - cluster1.client.mirror.1 - cluster1.client.mirror.2 - cluster1.client.mirror.3 - cluster1.client.mirror.4 - cluster1.client.mirror.5 - cluster1.client.mirror.6 - cluster2.client.mirror - cluster2.client.mirror.0 - cluster2.client.mirror.1 - cluster2.client.mirror.2 - cluster2.client.mirror.3 - cluster2.client.mirror.4 - cluster2.client.mirror.5 - cluster2.client.mirror.6 seed: 5839 sha1: 7f24e589328ae9e639c0009d93340e5b235d0a47 sleep_before_teardown: 0 subset: 99/128 suite: rbd suite_branch: main suite_path: /home/teuthworker/src/git.ceph.com_ceph_0d61375b8ab53be4c233f5b93e463edfdff9316a/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph.git suite_sha1: 0d61375b8ab53be4c233f5b93e463edfdff9316a targets: smithi135.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBJh1m5zJUpHrqEyHhqUVObJhAgM7iP05jGzbnT7mUeekySVDe8iX2BAY4xdaDHj9DekK4x3kUZUrLQKh3KfaYjY= smithi164.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBNmOzx19qAeUG8w7yLVIdwoPWkyQAh2DMyIlga3wbcZH8cBrjHKdqq+xguBx2x/IgM0PT6Rzgxf2HcPAt7MdHzI= tasks: - install: extra_packages: - rbd-mirror - ceph: cluster: cluster1 - ceph: cluster: cluster2 - exec: cluster1.client.mirror.0: - sudo ceph --cluster cluster1 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd' - rbd-mirror: client: cluster1.client.mirror.0 thrash: true - rbd-mirror: client: cluster1.client.mirror.1 thrash: true - rbd-mirror: client: cluster1.client.mirror.2 thrash: true - rbd-mirror: client: cluster1.client.mirror.3 thrash: true - rbd-mirror: client: cluster2.client.mirror.0 thrash: true - rbd-mirror: client: cluster2.client.mirror.1 thrash: true - rbd-mirror: client: cluster2.client.mirror.2 thrash: true - rbd-mirror: client: cluster2.client.mirror.3 thrash: true - rbd-mirror-thrash: cluster: cluster1 - rbd-mirror-thrash: cluster: cluster2 - workunit: clients: cluster1.client.mirror: - rbd/rbd_mirror_stress.sh env: CEPH_ARGS: '' MIRROR_POOL_MODE: image RBD_IMAGE_FEATURES: layering RBD_MIRROR_INSTANCES: '4' RBD_MIRROR_MODE: snapshot RBD_MIRROR_USE_EXISTING_CLUSTER: '1' RBD_MIRROR_USE_RBD_MIRROR: '1' timeout: 6h teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a timestamp: 2024-09-10_20:16:17 tube: smithi user: teuthology verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.3401856 2024-09-14T08:22:14.592 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph_0d61375b8ab53be4c233f5b93e463edfdff9316a/qa; will attempt to use it 2024-09-14T08:22:14.592 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph_0d61375b8ab53be4c233f5b93e463edfdff9316a/qa/tasks 2024-09-14T08:22:14.593 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-09-14T08:22:14.594 INFO:teuthology.task.internal:Checking packages... 2024-09-14T08:22:14.615 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash '7f24e589328ae9e639c0009d93340e5b235d0a47' 2024-09-14T08:22:14.615 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-14T08:22:14.616 INFO:teuthology.packaging:ref: None 2024-09-14T08:22:14.616 INFO:teuthology.packaging:tag: None 2024-09-14T08:22:14.616 INFO:teuthology.packaging:branch: main 2024-09-14T08:22:14.616 INFO:teuthology.packaging:sha1: 7f24e589328ae9e639c0009d93340e5b235d0a47 2024-09-14T08:22:14.616 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=main 2024-09-14T08:22:14.837 INFO:teuthology.task.internal:Found packages for ceph version 19.3.0-4926.g1c941a66 2024-09-14T08:22:14.839 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-09-14T08:22:14.846 INFO:teuthology.task.internal:no buildpackages task found 2024-09-14T08:22:14.847 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-09-14T08:22:14.871 INFO:teuthology.task.internal:Saving configuration 2024-09-14T08:22:14.892 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-09-14T08:22:14.899 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-09-14T08:22:14.930 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi135.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-09-10_20:16:17-rbd-main-distro-default-smithi/7900492', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-09-14 08:17:04.576462', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBJh1m5zJUpHrqEyHhqUVObJhAgM7iP05jGzbnT7mUeekySVDe8iX2BAY4xdaDHj9DekK4x3kUZUrLQKh3KfaYjY='} 2024-09-14T08:22:14.951 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi164.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-09-10_20:16:17-rbd-main-distro-default-smithi/7900492', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-09-14 08:17:04.578440', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBNmOzx19qAeUG8w7yLVIdwoPWkyQAh2DMyIlga3wbcZH8cBrjHKdqq+xguBx2x/IgM0PT6Rzgxf2HcPAt7MdHzI='} 2024-09-14T08:22:14.951 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-09-14T08:22:14.958 INFO:teuthology.task.internal:roles: ubuntu@smithi135.front.sepia.ceph.com - ['cluster1.mon.a', 'cluster1.mgr.x', 'cluster2.mgr.x', 'cluster1.osd.0', 'cluster1.osd.1', 'cluster1.osd.2', 'cluster1.client.0', 'cluster2.client.0'] 2024-09-14T08:22:14.958 INFO:teuthology.task.internal:roles: ubuntu@smithi164.front.sepia.ceph.com - ['cluster2.mon.a', 'cluster2.osd.0', 'cluster2.osd.1', 'cluster2.osd.2', 'cluster1.client.mirror', 'cluster1.client.mirror.0', 'cluster1.client.mirror.1', 'cluster1.client.mirror.2', 'cluster1.client.mirror.3', 'cluster1.client.mirror.4', 'cluster1.client.mirror.5', 'cluster1.client.mirror.6', 'cluster2.client.mirror', 'cluster2.client.mirror.0', 'cluster2.client.mirror.1', 'cluster2.client.mirror.2', 'cluster2.client.mirror.3', 'cluster2.client.mirror.4', 'cluster2.client.mirror.5', 'cluster2.client.mirror.6'] 2024-09-14T08:22:14.958 INFO:teuthology.run_tasks:Running task console_log... 2024-09-14T08:22:15.079 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f773fd5fbe0>, signals=[15]) 2024-09-14T08:22:15.079 INFO:teuthology.run_tasks:Running task internal.connect... 2024-09-14T08:22:15.086 INFO:teuthology.task.internal:Opening connections... 2024-09-14T08:22:15.086 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi135.front.sepia.ceph.com 2024-09-14T08:22:15.087 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi135.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-14T08:22:15.176 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi164.front.sepia.ceph.com 2024-09-14T08:22:15.177 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi164.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-14T08:22:15.249 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-09-14T08:22:15.256 DEBUG:teuthology.orchestra.run.smithi135:> uname -m 2024-09-14T08:22:15.276 INFO:teuthology.orchestra.run.smithi135.stdout:x86_64 2024-09-14T08:22:15.276 DEBUG:teuthology.orchestra.run.smithi135:> cat /etc/os-release 2024-09-14T08:22:15.333 INFO:teuthology.orchestra.run.smithi135.stdout:NAME="CentOS Stream" 2024-09-14T08:22:15.333 INFO:teuthology.orchestra.run.smithi135.stdout:VERSION="9" 2024-09-14T08:22:15.333 INFO:teuthology.orchestra.run.smithi135.stdout:ID="centos" 2024-09-14T08:22:15.333 INFO:teuthology.orchestra.run.smithi135.stdout:ID_LIKE="rhel fedora" 2024-09-14T08:22:15.333 INFO:teuthology.orchestra.run.smithi135.stdout:VERSION_ID="9" 2024-09-14T08:22:15.333 INFO:teuthology.orchestra.run.smithi135.stdout:PLATFORM_ID="platform:el9" 2024-09-14T08:22:15.333 INFO:teuthology.orchestra.run.smithi135.stdout:PRETTY_NAME="CentOS Stream 9" 2024-09-14T08:22:15.333 INFO:teuthology.orchestra.run.smithi135.stdout:ANSI_COLOR="0;31" 2024-09-14T08:22:15.334 INFO:teuthology.orchestra.run.smithi135.stdout:LOGO="fedora-logo-icon" 2024-09-14T08:22:15.334 INFO:teuthology.orchestra.run.smithi135.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-09-14T08:22:15.334 INFO:teuthology.orchestra.run.smithi135.stdout:HOME_URL="https://centos.org/" 2024-09-14T08:22:15.334 INFO:teuthology.orchestra.run.smithi135.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2024-09-14T08:22:15.334 INFO:teuthology.orchestra.run.smithi135.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-09-14T08:22:15.334 INFO:teuthology.orchestra.run.smithi135.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-09-14T08:22:15.335 INFO:teuthology.lock.ops:Updating smithi135.front.sepia.ceph.com on lock server 2024-09-14T08:22:15.356 DEBUG:teuthology.orchestra.run.smithi164:> uname -m 2024-09-14T08:22:15.374 INFO:teuthology.orchestra.run.smithi164.stdout:x86_64 2024-09-14T08:22:15.374 DEBUG:teuthology.orchestra.run.smithi164:> cat /etc/os-release 2024-09-14T08:22:15.433 INFO:teuthology.orchestra.run.smithi164.stdout:NAME="CentOS Stream" 2024-09-14T08:22:15.433 INFO:teuthology.orchestra.run.smithi164.stdout:VERSION="9" 2024-09-14T08:22:15.433 INFO:teuthology.orchestra.run.smithi164.stdout:ID="centos" 2024-09-14T08:22:15.433 INFO:teuthology.orchestra.run.smithi164.stdout:ID_LIKE="rhel fedora" 2024-09-14T08:22:15.433 INFO:teuthology.orchestra.run.smithi164.stdout:VERSION_ID="9" 2024-09-14T08:22:15.433 INFO:teuthology.orchestra.run.smithi164.stdout:PLATFORM_ID="platform:el9" 2024-09-14T08:22:15.433 INFO:teuthology.orchestra.run.smithi164.stdout:PRETTY_NAME="CentOS Stream 9" 2024-09-14T08:22:15.433 INFO:teuthology.orchestra.run.smithi164.stdout:ANSI_COLOR="0;31" 2024-09-14T08:22:15.433 INFO:teuthology.orchestra.run.smithi164.stdout:LOGO="fedora-logo-icon" 2024-09-14T08:22:15.433 INFO:teuthology.orchestra.run.smithi164.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-09-14T08:22:15.433 INFO:teuthology.orchestra.run.smithi164.stdout:HOME_URL="https://centos.org/" 2024-09-14T08:22:15.434 INFO:teuthology.orchestra.run.smithi164.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2024-09-14T08:22:15.434 INFO:teuthology.orchestra.run.smithi164.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-09-14T08:22:15.434 INFO:teuthology.orchestra.run.smithi164.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-09-14T08:22:15.434 INFO:teuthology.lock.ops:Updating smithi164.front.sepia.ceph.com on lock server 2024-09-14T08:22:15.455 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-09-14T08:22:15.465 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-09-14T08:22:15.471 INFO:teuthology.task.internal:Checking for old test directory... 2024-09-14T08:22:15.471 DEBUG:teuthology.orchestra.run.smithi135:> test '!' -e /home/ubuntu/cephtest 2024-09-14T08:22:15.474 DEBUG:teuthology.orchestra.run.smithi164:> test '!' -e /home/ubuntu/cephtest 2024-09-14T08:22:15.490 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-09-14T08:22:15.497 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-09-14T08:22:15.497 DEBUG:teuthology.orchestra.run.smithi135:> test -z $(ls -A /var/lib/ceph) 2024-09-14T08:22:15.531 DEBUG:teuthology.orchestra.run.smithi164:> test -z $(ls -A /var/lib/ceph) 2024-09-14T08:22:15.573 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-09-14T08:22:15.623 INFO:teuthology.run_tasks:Running task kernel... 2024-09-14T08:22:15.638 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-09-14T08:22:15.638 INFO:teuthology.task.kernel:config {'cluster1.mon.a': {'kdb': 1, 'sha1': 'distro'}, 'cluster2.mon.a': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-09-14T08:22:15.638 DEBUG:teuthology.orchestra.run.smithi135:> test -f /run/.containerenv -o -f /.dockerenv 2024-09-14T08:22:15.639 DEBUG:teuthology.orchestra.run.smithi164:> test -f /run/.containerenv -o -f /.dockerenv 2024-09-14T08:22:15.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T08:22:15.656 DEBUG:teuthology.orchestra.run.smithi135:> uname -r 2024-09-14T08:22:15.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T08:22:15.657 DEBUG:teuthology.orchestra.run.smithi164:> uname -r 2024-09-14T08:22:15.713 INFO:teuthology.orchestra.run.smithi164.stdout:5.14.0-503.el9.x86_64 2024-09-14T08:22:15.714 INFO:teuthology.task.kernel:Running kernel on smithi164: 5.14.0-503.el9.x86_64 2024-09-14T08:22:15.714 DEBUG:teuthology.orchestra.run.smithi164:> sudo yum install -y kernel 2024-09-14T08:22:15.715 INFO:teuthology.orchestra.run.smithi135.stdout:5.14.0-503.el9.x86_64 2024-09-14T08:22:15.715 INFO:teuthology.task.kernel:Running kernel on smithi135: 5.14.0-503.el9.x86_64 2024-09-14T08:22:15.715 DEBUG:teuthology.orchestra.run.smithi135:> sudo yum install -y kernel 2024-09-14T08:22:18.095 INFO:teuthology.orchestra.run.smithi135.stdout:CentOS Stream 9 - BaseOS 14 MB/s | 8.2 MB 00:00 2024-09-14T08:22:19.887 INFO:teuthology.orchestra.run.smithi135.stdout:CentOS Stream 9 - AppStream 29 MB/s | 20 MB 00:00 2024-09-14T08:22:22.521 INFO:teuthology.orchestra.run.smithi164.stdout:CentOS Stream 9 - BaseOS 1.6 MB/s | 8.2 MB 00:05 2024-09-14T08:22:24.135 INFO:teuthology.orchestra.run.smithi135.stdout:CentOS Stream 9 - CRB 15 MB/s | 6.5 MB 00:00 2024-09-14T08:22:24.747 INFO:teuthology.orchestra.run.smithi164.stdout:CentOS Stream 9 - AppStream 18 MB/s | 20 MB 00:01 2024-09-14T08:22:25.523 INFO:teuthology.orchestra.run.smithi135.stdout:CentOS Stream 9 - Extras packages 120 kB/s | 19 kB 00:00 2024-09-14T08:22:26.594 INFO:teuthology.orchestra.run.smithi135.stdout:Extra Packages for Enterprise Linux 24 MB/s | 23 MB 00:00 2024-09-14T08:22:30.156 INFO:teuthology.orchestra.run.smithi164.stdout:CentOS Stream 9 - CRB 4.0 MB/s | 6.5 MB 00:01 2024-09-14T08:22:31.589 INFO:teuthology.orchestra.run.smithi164.stdout:CentOS Stream 9 - Extras packages 80 kB/s | 19 kB 00:00 2024-09-14T08:22:32.208 INFO:teuthology.orchestra.run.smithi135.stdout:lab-extras 15 kB/s | 1.7 kB 00:00 2024-09-14T08:22:32.469 INFO:teuthology.orchestra.run.smithi164.stdout:Extra Packages for Enterprise Linux 30 MB/s | 23 MB 00:00 2024-09-14T08:22:34.225 INFO:teuthology.orchestra.run.smithi135.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-09-14T08:22:34.225 INFO:teuthology.orchestra.run.smithi135.stdout:Package kernel-5.14.0-503.el9.x86_64 is already installed. 2024-09-14T08:22:34.336 INFO:teuthology.orchestra.run.smithi135.stdout:Dependencies resolved. 2024-09-14T08:22:34.343 INFO:teuthology.orchestra.run.smithi135.stdout:================================================================================ 2024-09-14T08:22:34.343 INFO:teuthology.orchestra.run.smithi135.stdout: Package Architecture Version Repository Size 2024-09-14T08:22:34.343 INFO:teuthology.orchestra.run.smithi135.stdout:================================================================================ 2024-09-14T08:22:34.343 INFO:teuthology.orchestra.run.smithi135.stdout:Installing: 2024-09-14T08:22:34.343 INFO:teuthology.orchestra.run.smithi135.stdout: kernel x86_64 5.14.0-505.el9 baseos 23 k 2024-09-14T08:22:34.343 INFO:teuthology.orchestra.run.smithi135.stdout:Installing dependencies: 2024-09-14T08:22:34.343 INFO:teuthology.orchestra.run.smithi135.stdout: kernel-core x86_64 5.14.0-505.el9 baseos 16 M 2024-09-14T08:22:34.343 INFO:teuthology.orchestra.run.smithi135.stdout: kernel-modules x86_64 5.14.0-505.el9 baseos 35 M 2024-09-14T08:22:34.343 INFO:teuthology.orchestra.run.smithi135.stdout: kernel-modules-core x86_64 5.14.0-505.el9 baseos 29 M 2024-09-14T08:22:34.343 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:22:34.343 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction Summary 2024-09-14T08:22:34.344 INFO:teuthology.orchestra.run.smithi135.stdout:================================================================================ 2024-09-14T08:22:34.344 INFO:teuthology.orchestra.run.smithi135.stdout:Install 4 Packages 2024-09-14T08:22:34.344 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:22:34.344 INFO:teuthology.orchestra.run.smithi135.stdout:Total download size: 79 M 2024-09-14T08:22:34.344 INFO:teuthology.orchestra.run.smithi135.stdout:Installed size: 126 M 2024-09-14T08:22:34.344 INFO:teuthology.orchestra.run.smithi135.stdout:Downloading Packages: 2024-09-14T08:22:34.453 INFO:teuthology.orchestra.run.smithi135.stdout:(1/4): kernel-5.14.0-505.el9.x86_64.rpm 255 kB/s | 23 kB 00:00 2024-09-14T08:22:37.512 INFO:teuthology.orchestra.run.smithi135.stdout:(2/4): kernel-core-5.14.0-505.el9.x86_64.rpm 5.0 MB/s | 16 MB 00:03 2024-09-14T08:22:37.916 INFO:teuthology.orchestra.run.smithi164.stdout:lab-extras 46 kB/s | 1.7 kB 00:00 2024-09-14T08:22:39.931 INFO:teuthology.orchestra.run.smithi164.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-09-14T08:22:39.931 INFO:teuthology.orchestra.run.smithi164.stdout:Package kernel-5.14.0-503.el9.x86_64 is already installed. 2024-09-14T08:22:39.953 INFO:teuthology.orchestra.run.smithi135.stdout:(3/4): kernel-modules-core-5.14.0-505.el9.x86_6 5.2 MB/s | 29 MB 00:05 2024-09-14T08:22:40.034 INFO:teuthology.orchestra.run.smithi164.stdout:Dependencies resolved. 2024-09-14T08:22:40.042 INFO:teuthology.orchestra.run.smithi164.stdout:================================================================================ 2024-09-14T08:22:40.042 INFO:teuthology.orchestra.run.smithi164.stdout: Package Architecture Version Repository Size 2024-09-14T08:22:40.042 INFO:teuthology.orchestra.run.smithi164.stdout:================================================================================ 2024-09-14T08:22:40.042 INFO:teuthology.orchestra.run.smithi164.stdout:Installing: 2024-09-14T08:22:40.042 INFO:teuthology.orchestra.run.smithi164.stdout: kernel x86_64 5.14.0-505.el9 baseos 23 k 2024-09-14T08:22:40.042 INFO:teuthology.orchestra.run.smithi164.stdout:Installing dependencies: 2024-09-14T08:22:40.042 INFO:teuthology.orchestra.run.smithi164.stdout: kernel-core x86_64 5.14.0-505.el9 baseos 16 M 2024-09-14T08:22:40.043 INFO:teuthology.orchestra.run.smithi164.stdout: kernel-modules x86_64 5.14.0-505.el9 baseos 35 M 2024-09-14T08:22:40.043 INFO:teuthology.orchestra.run.smithi164.stdout: kernel-modules-core x86_64 5.14.0-505.el9 baseos 29 M 2024-09-14T08:22:40.043 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:22:40.043 INFO:teuthology.orchestra.run.smithi164.stdout:Transaction Summary 2024-09-14T08:22:40.043 INFO:teuthology.orchestra.run.smithi164.stdout:================================================================================ 2024-09-14T08:22:40.043 INFO:teuthology.orchestra.run.smithi164.stdout:Install 4 Packages 2024-09-14T08:22:40.043 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:22:40.044 INFO:teuthology.orchestra.run.smithi164.stdout:Total download size: 79 M 2024-09-14T08:22:40.044 INFO:teuthology.orchestra.run.smithi164.stdout:Installed size: 126 M 2024-09-14T08:22:40.044 INFO:teuthology.orchestra.run.smithi164.stdout:Downloading Packages: 2024-09-14T08:22:40.142 INFO:teuthology.orchestra.run.smithi164.stdout:(1/4): kernel-5.14.0-505.el9.x86_64.rpm 295 kB/s | 23 kB 00:00 2024-09-14T08:22:40.312 INFO:teuthology.orchestra.run.smithi135.stdout:(4/4): kernel-modules-5.14.0-505.el9.x86_64.rpm 5.8 MB/s | 35 MB 00:05 2024-09-14T08:22:40.312 INFO:teuthology.orchestra.run.smithi135.stdout:-------------------------------------------------------------------------------- 2024-09-14T08:22:40.312 INFO:teuthology.orchestra.run.smithi135.stdout:Total 13 MB/s | 79 MB 00:05 2024-09-14T08:22:40.684 INFO:teuthology.orchestra.run.smithi164.stdout:(2/4): kernel-core-5.14.0-505.el9.x86_64.rpm 25 MB/s | 16 MB 00:00 2024-09-14T08:22:40.998 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction check 2024-09-14T08:22:41.160 INFO:teuthology.orchestra.run.smithi164.stdout:(3/4): kernel-modules-5.14.0-505.el9.x86_64.rpm 32 MB/s | 35 MB 00:01 2024-09-14T08:22:41.371 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction check succeeded. 2024-09-14T08:22:41.371 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction test 2024-09-14T08:22:41.577 INFO:teuthology.orchestra.run.smithi164.stdout:(4/4): kernel-modules-core-5.14.0-505.el9.x86_6 20 MB/s | 29 MB 00:01 2024-09-14T08:22:41.577 INFO:teuthology.orchestra.run.smithi164.stdout:-------------------------------------------------------------------------------- 2024-09-14T08:22:41.577 INFO:teuthology.orchestra.run.smithi164.stdout:Total 51 MB/s | 79 MB 00:01 2024-09-14T08:22:42.261 INFO:teuthology.orchestra.run.smithi164.stdout:Running transaction check 2024-09-14T08:22:42.368 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction test succeeded. 2024-09-14T08:22:42.369 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction 2024-09-14T08:22:42.590 INFO:teuthology.orchestra.run.smithi164.stdout:Transaction check succeeded. 2024-09-14T08:22:42.590 INFO:teuthology.orchestra.run.smithi164.stdout:Running transaction test 2024-09-14T08:22:43.539 INFO:teuthology.orchestra.run.smithi164.stdout:Transaction test succeeded. 2024-09-14T08:22:43.539 INFO:teuthology.orchestra.run.smithi164.stdout:Running transaction 2024-09-14T08:22:44.803 INFO:teuthology.orchestra.run.smithi135.stdout: Preparing : 1/1 2024-09-14T08:22:45.512 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : kernel-modules-core-5.14.0-505.el9.x86_64 1/4 2024-09-14T08:22:45.612 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-14T08:22:45.990 INFO:teuthology.orchestra.run.smithi164.stdout: Preparing : 1/1 2024-09-14T08:22:46.628 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : kernel-modules-core-5.14.0-505.el9.x86_64 1/4 2024-09-14T08:22:46.726 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-14T08:22:48.403 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-14T08:22:48.538 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-14T08:22:49.553 INFO:teuthology.orchestra.run.smithi164.stdout: Running scriptlet: kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-14T08:22:49.680 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-14T08:22:53.398 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-14T08:22:53.654 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : kernel-5.14.0-505.el9.x86_64 4/4 2024-09-14T08:22:54.498 INFO:teuthology.orchestra.run.smithi164.stdout: Running scriptlet: kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-14T08:22:54.530 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : kernel-5.14.0-505.el9.x86_64 4/4 2024-09-14T08:22:58.516 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: kernel-modules-core-5.14.0-505.el9.x86_64 4/4 2024-09-14T08:22:59.414 INFO:teuthology.orchestra.run.smithi164.stdout: Running scriptlet: kernel-modules-core-5.14.0-505.el9.x86_64 4/4 2024-09-14T08:23:37.246 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: kernel-core-5.14.0-505.el9.x86_64 4/4 2024-09-14T08:23:37.285 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: kernel-modules-5.14.0-505.el9.x86_64 4/4 2024-09-14T08:23:37.671 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: kernel-5.14.0-505.el9.x86_64 4/4 2024-09-14T08:23:37.671 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : kernel-5.14.0-505.el9.x86_64 1/4 2024-09-14T08:23:37.671 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-14T08:23:37.671 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-14T08:23:37.755 INFO:teuthology.orchestra.run.smithi164.stdout: Running scriptlet: kernel-core-5.14.0-505.el9.x86_64 4/4 2024-09-14T08:23:37.795 INFO:teuthology.orchestra.run.smithi164.stdout: Running scriptlet: kernel-modules-5.14.0-505.el9.x86_64 4/4 2024-09-14T08:23:38.148 INFO:teuthology.orchestra.run.smithi164.stdout: Running scriptlet: kernel-5.14.0-505.el9.x86_64 4/4 2024-09-14T08:23:38.148 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : kernel-5.14.0-505.el9.x86_64 1/4 2024-09-14T08:23:38.148 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-14T08:23:38.148 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-14T08:23:38.548 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : kernel-modules-core-5.14.0-505.el9.x86_64 4/4 2024-09-14T08:23:38.548 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:23:38.548 INFO:teuthology.orchestra.run.smithi135.stdout:Installed: 2024-09-14T08:23:38.548 INFO:teuthology.orchestra.run.smithi135.stdout: kernel-5.14.0-505.el9.x86_64 2024-09-14T08:23:38.548 INFO:teuthology.orchestra.run.smithi135.stdout: kernel-core-5.14.0-505.el9.x86_64 2024-09-14T08:23:38.548 INFO:teuthology.orchestra.run.smithi135.stdout: kernel-modules-5.14.0-505.el9.x86_64 2024-09-14T08:23:38.548 INFO:teuthology.orchestra.run.smithi135.stdout: kernel-modules-core-5.14.0-505.el9.x86_64 2024-09-14T08:23:38.548 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:23:38.548 INFO:teuthology.orchestra.run.smithi135.stdout:Complete! 2024-09-14T08:23:38.932 DEBUG:teuthology.orchestra.run.smithi135:> rpm -q kernel | sort -rV | head -n 1 2024-09-14T08:23:39.014 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : kernel-modules-core-5.14.0-505.el9.x86_64 4/4 2024-09-14T08:23:39.014 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:23:39.014 INFO:teuthology.orchestra.run.smithi164.stdout:Installed: 2024-09-14T08:23:39.014 INFO:teuthology.orchestra.run.smithi164.stdout: kernel-5.14.0-505.el9.x86_64 2024-09-14T08:23:39.014 INFO:teuthology.orchestra.run.smithi164.stdout: kernel-core-5.14.0-505.el9.x86_64 2024-09-14T08:23:39.014 INFO:teuthology.orchestra.run.smithi164.stdout: kernel-modules-5.14.0-505.el9.x86_64 2024-09-14T08:23:39.014 INFO:teuthology.orchestra.run.smithi164.stdout: kernel-modules-core-5.14.0-505.el9.x86_64 2024-09-14T08:23:39.014 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:23:39.014 INFO:teuthology.orchestra.run.smithi164.stdout:Complete! 2024-09-14T08:23:39.020 INFO:teuthology.orchestra.run.smithi135.stdout:kernel-5.14.0-505.el9.x86_64 2024-09-14T08:23:39.020 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-505.el9.x86_64 2024-09-14T08:23:39.020 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-503.el9.x86_64 Expected: 5.14.0-505.el9.x86_64 2024-09-14T08:23:39.021 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-09-14T08:23:39.021 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-09-14T08:23:39.021 INFO:teuthology.task.kernel:Installing distro kernel on cluster1.mon.a... 2024-09-14T08:23:39.021 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi135.front.sepia.ceph.com, path=None, version=distro) 2024-09-14T08:23:39.021 DEBUG:teuthology.orchestra.run.smithi135:> rpm -q kernel | sort -rV | head -n 1 2024-09-14T08:23:39.080 INFO:teuthology.orchestra.run.smithi135.stdout:kernel-5.14.0-505.el9.x86_64 2024-09-14T08:23:39.080 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-505.el9.x86_64 2024-09-14T08:23:39.080 DEBUG:teuthology.orchestra.run.smithi135:> sudo rpm -qi grub2-tools 2024-09-14T08:23:39.112 INFO:teuthology.orchestra.run.smithi135.stdout:Name : grub2-tools 2024-09-14T08:23:39.112 INFO:teuthology.orchestra.run.smithi135.stdout:Epoch : 1 2024-09-14T08:23:39.112 INFO:teuthology.orchestra.run.smithi135.stdout:Version : 2.06 2024-09-14T08:23:39.112 INFO:teuthology.orchestra.run.smithi135.stdout:Release : 82.el9 2024-09-14T08:23:39.112 INFO:teuthology.orchestra.run.smithi135.stdout:Architecture: x86_64 2024-09-14T08:23:39.112 INFO:teuthology.orchestra.run.smithi135.stdout:Install Date: Wed 28 Aug 2024 09:06:27 PM UTC 2024-09-14T08:23:39.112 INFO:teuthology.orchestra.run.smithi135.stdout:Group : Unspecified 2024-09-14T08:23:39.112 INFO:teuthology.orchestra.run.smithi135.stdout:Size : 8274670 2024-09-14T08:23:39.112 INFO:teuthology.orchestra.run.smithi135.stdout:License : GPLv3+ 2024-09-14T08:23:39.112 INFO:teuthology.orchestra.run.smithi135.stdout:Signature : RSA/SHA256, Mon 01 Jul 2024 07:46:42 AM UTC, Key ID 05b555b38483c65d 2024-09-14T08:23:39.112 INFO:teuthology.orchestra.run.smithi135.stdout:Source RPM : grub2-2.06-82.el9.src.rpm 2024-09-14T08:23:39.113 INFO:teuthology.orchestra.run.smithi135.stdout:Build Date : Fri 28 Jun 2024 09:24:44 AM UTC 2024-09-14T08:23:39.113 INFO:teuthology.orchestra.run.smithi135.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-09-14T08:23:39.113 INFO:teuthology.orchestra.run.smithi135.stdout:Packager : builder@centos.org 2024-09-14T08:23:39.113 INFO:teuthology.orchestra.run.smithi135.stdout:Vendor : CentOS 2024-09-14T08:23:39.113 INFO:teuthology.orchestra.run.smithi135.stdout:URL : http://www.gnu.org/software/grub/ 2024-09-14T08:23:39.113 INFO:teuthology.orchestra.run.smithi135.stdout:Summary : Support tools for GRUB. 2024-09-14T08:23:39.113 INFO:teuthology.orchestra.run.smithi135.stdout:Description : 2024-09-14T08:23:39.113 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:23:39.113 INFO:teuthology.orchestra.run.smithi135.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-09-14T08:23:39.113 INFO:teuthology.orchestra.run.smithi135.stdout:customizable bootloader with modular architecture. It supports a rich 2024-09-14T08:23:39.113 INFO:teuthology.orchestra.run.smithi135.stdout:variety of kernel formats, file systems, computer architectures and 2024-09-14T08:23:39.113 INFO:teuthology.orchestra.run.smithi135.stdout:hardware devices. 2024-09-14T08:23:39.113 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:23:39.113 INFO:teuthology.orchestra.run.smithi135.stdout:This subpackage provides tools for support of all platforms. 2024-09-14T08:23:39.114 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-09-14T08:23:39.114 INFO:teuthology.task.kernel:Updating grub on smithi135 to boot 5.14.0-505.el9.x86_64 2024-09-14T08:23:39.115 DEBUG:teuthology.orchestra.run.smithi135:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-09-14T08:23:39.367 DEBUG:teuthology.orchestra.run.smithi164:> rpm -q kernel | sort -rV | head -n 1 2024-09-14T08:23:39.462 INFO:teuthology.orchestra.run.smithi164.stdout:kernel-5.14.0-505.el9.x86_64 2024-09-14T08:23:39.463 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-505.el9.x86_64 2024-09-14T08:23:39.463 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-503.el9.x86_64 Expected: 5.14.0-505.el9.x86_64 2024-09-14T08:23:39.463 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-09-14T08:23:39.463 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-09-14T08:23:39.463 INFO:teuthology.task.kernel:Installing distro kernel on cluster2.mon.a... 2024-09-14T08:23:39.463 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi164.front.sepia.ceph.com, path=None, version=distro) 2024-09-14T08:23:39.463 DEBUG:teuthology.orchestra.run.smithi164:> rpm -q kernel | sort -rV | head -n 1 2024-09-14T08:23:39.522 INFO:teuthology.orchestra.run.smithi164.stdout:kernel-5.14.0-505.el9.x86_64 2024-09-14T08:23:39.522 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-505.el9.x86_64 2024-09-14T08:23:39.522 DEBUG:teuthology.orchestra.run.smithi164:> sudo rpm -qi grub2-tools 2024-09-14T08:23:39.554 INFO:teuthology.orchestra.run.smithi164.stdout:Name : grub2-tools 2024-09-14T08:23:39.554 INFO:teuthology.orchestra.run.smithi164.stdout:Epoch : 1 2024-09-14T08:23:39.554 INFO:teuthology.orchestra.run.smithi164.stdout:Version : 2.06 2024-09-14T08:23:39.555 INFO:teuthology.orchestra.run.smithi164.stdout:Release : 82.el9 2024-09-14T08:23:39.555 INFO:teuthology.orchestra.run.smithi164.stdout:Architecture: x86_64 2024-09-14T08:23:39.555 INFO:teuthology.orchestra.run.smithi164.stdout:Install Date: Wed 28 Aug 2024 09:06:27 PM UTC 2024-09-14T08:23:39.555 INFO:teuthology.orchestra.run.smithi164.stdout:Group : Unspecified 2024-09-14T08:23:39.555 INFO:teuthology.orchestra.run.smithi164.stdout:Size : 8274670 2024-09-14T08:23:39.555 INFO:teuthology.orchestra.run.smithi164.stdout:License : GPLv3+ 2024-09-14T08:23:39.555 INFO:teuthology.orchestra.run.smithi164.stdout:Signature : RSA/SHA256, Mon 01 Jul 2024 07:46:42 AM UTC, Key ID 05b555b38483c65d 2024-09-14T08:23:39.555 INFO:teuthology.orchestra.run.smithi164.stdout:Source RPM : grub2-2.06-82.el9.src.rpm 2024-09-14T08:23:39.555 INFO:teuthology.orchestra.run.smithi164.stdout:Build Date : Fri 28 Jun 2024 09:24:44 AM UTC 2024-09-14T08:23:39.555 INFO:teuthology.orchestra.run.smithi164.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-09-14T08:23:39.555 INFO:teuthology.orchestra.run.smithi164.stdout:Packager : builder@centos.org 2024-09-14T08:23:39.555 INFO:teuthology.orchestra.run.smithi164.stdout:Vendor : CentOS 2024-09-14T08:23:39.555 INFO:teuthology.orchestra.run.smithi164.stdout:URL : http://www.gnu.org/software/grub/ 2024-09-14T08:23:39.555 INFO:teuthology.orchestra.run.smithi164.stdout:Summary : Support tools for GRUB. 2024-09-14T08:23:39.556 INFO:teuthology.orchestra.run.smithi164.stdout:Description : 2024-09-14T08:23:39.556 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:23:39.556 INFO:teuthology.orchestra.run.smithi164.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-09-14T08:23:39.556 INFO:teuthology.orchestra.run.smithi164.stdout:customizable bootloader with modular architecture. It supports a rich 2024-09-14T08:23:39.556 INFO:teuthology.orchestra.run.smithi164.stdout:variety of kernel formats, file systems, computer architectures and 2024-09-14T08:23:39.556 INFO:teuthology.orchestra.run.smithi164.stdout:hardware devices. 2024-09-14T08:23:39.556 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:23:39.556 INFO:teuthology.orchestra.run.smithi164.stdout:This subpackage provides tools for support of all platforms. 2024-09-14T08:23:39.557 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-09-14T08:23:39.557 INFO:teuthology.task.kernel:Updating grub on smithi164 to boot 5.14.0-505.el9.x86_64 2024-09-14T08:23:39.557 DEBUG:teuthology.orchestra.run.smithi164:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-09-14T08:23:39.768 INFO:teuthology.orchestra.run.smithi135.stderr:Generating grub configuration file ... 2024-09-14T08:23:40.227 INFO:teuthology.orchestra.run.smithi164.stderr:Generating grub configuration file ... 2024-09-14T08:23:41.142 INFO:teuthology.orchestra.run.smithi135.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-09-14T08:23:41.158 INFO:teuthology.orchestra.run.smithi135.stderr:done 2024-09-14T08:23:41.160 DEBUG:teuthology.orchestra.run.smithi135:> mktemp 2024-09-14T08:23:41.176 INFO:teuthology.orchestra.run.smithi135.stdout:/tmp/tmp.yruxQCCVG1 2024-09-14T08:23:41.176 DEBUG:teuthology.orchestra.run.smithi135:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.yruxQCCVG1 2024-09-14T08:23:41.249 DEBUG:teuthology.orchestra.run.smithi135:> sudo chmod 0666 /tmp/tmp.yruxQCCVG1 2024-09-14T08:23:41.397 DEBUG:teuthology.orchestra.remote:smithi135:/tmp/tmp.yruxQCCVG1 is 6KB 2024-09-14T08:23:41.419 DEBUG:teuthology.orchestra.run.smithi135:> rm -fr /tmp/tmp.yruxQCCVG1 2024-09-14T08:23:41.434 DEBUG:teuthology.orchestra.run.smithi135:> sudo /bin/ls /boot/loader/entries || true 2024-09-14T08:23:41.498 INFO:teuthology.orchestra.run.smithi135.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-0-rescue.conf 2024-09-14T08:23:41.498 INFO:teuthology.orchestra.run.smithi135.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-239.el9.x86_64.conf 2024-09-14T08:23:41.498 INFO:teuthology.orchestra.run.smithi135.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-503.el9.x86_64.conf 2024-09-14T08:23:41.498 INFO:teuthology.orchestra.run.smithi135.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-505.el9.x86_64.conf 2024-09-14T08:23:41.499 DEBUG:teuthology.orchestra.run.smithi135:> sudo grub2-set-default fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-505.el9.x86_64 2024-09-14T08:23:41.570 INFO:teuthology.orchestra.run.smithi164.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-09-14T08:23:41.597 INFO:teuthology.orchestra.run.smithi164.stderr:done 2024-09-14T08:23:41.599 DEBUG:teuthology.orchestra.run.smithi164:> mktemp 2024-09-14T08:23:41.614 INFO:teuthology.orchestra.run.smithi164.stdout:/tmp/tmp.sUmcLrBCIw 2024-09-14T08:23:41.615 DEBUG:teuthology.orchestra.run.smithi164:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.sUmcLrBCIw 2024-09-14T08:23:41.654 DEBUG:teuthology.orchestra.run.smithi135:> sudo shutdown -r now 2024-09-14T08:23:41.682 DEBUG:teuthology.orchestra.run.smithi164:> sudo chmod 0666 /tmp/tmp.sUmcLrBCIw 2024-09-14T08:23:41.755 ERROR:paramiko.transport:Socket exception: Connection reset by peer (104) 2024-09-14T08:23:41.824 DEBUG:teuthology.orchestra.remote:smithi164:/tmp/tmp.sUmcLrBCIw is 6KB 2024-09-14T08:23:41.846 DEBUG:teuthology.orchestra.run.smithi164:> rm -fr /tmp/tmp.sUmcLrBCIw 2024-09-14T08:23:41.861 DEBUG:teuthology.orchestra.run.smithi164:> sudo /bin/ls /boot/loader/entries || true 2024-09-14T08:23:41.924 INFO:teuthology.orchestra.run.smithi164.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-0-rescue.conf 2024-09-14T08:23:41.924 INFO:teuthology.orchestra.run.smithi164.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-239.el9.x86_64.conf 2024-09-14T08:23:41.924 INFO:teuthology.orchestra.run.smithi164.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-503.el9.x86_64.conf 2024-09-14T08:23:41.924 INFO:teuthology.orchestra.run.smithi164.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-505.el9.x86_64.conf 2024-09-14T08:23:41.925 DEBUG:teuthology.orchestra.run.smithi164:> sudo grub2-set-default fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-505.el9.x86_64 2024-09-14T08:23:42.080 DEBUG:teuthology.orchestra.run.smithi164:> sudo shutdown -r now 2024-09-14T08:24:11.687 INFO:teuthology.task.kernel:Checking client cluster1.mon.a for new kernel version... 2024-09-14T08:24:11.687 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi135.front.sepia.ceph.com' 2024-09-14T08:24:11.688 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi135.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-14T08:24:12.084 INFO:teuthology.task.kernel:Checking client cluster2.mon.a for new kernel version... 2024-09-14T08:24:12.084 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi164.front.sepia.ceph.com' 2024-09-14T08:24:12.085 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi164.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-14T08:24:30.013 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.135 2024-09-14T08:24:30.525 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.164 2024-09-14T08:24:39.019 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi135.front.sepia.ceph.com' 2024-09-14T08:24:39.021 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi135.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-14T08:24:39.526 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi164.front.sepia.ceph.com' 2024-09-14T08:24:39.527 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi164.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-14T08:24:42.077 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.135 2024-09-14T08:24:42.589 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.164 2024-09-14T08:24:54.089 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi135.front.sepia.ceph.com' 2024-09-14T08:24:54.090 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi135.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-14T08:24:54.590 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi164.front.sepia.ceph.com' 2024-09-14T08:24:54.591 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi164.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-14T08:25:25.886 DEBUG:teuthology.orchestra.run.smithi135:> true 2024-09-14T08:25:26.226 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi135.front.sepia.ceph.com' 2024-09-14T08:25:26.227 INFO:teuthology.task.kernel:Checking kernel version of cluster1.mon.a, want "5.14.0-505.el9.x86_64"... 2024-09-14T08:25:26.227 DEBUG:teuthology.orchestra.run.smithi135:> uname -r 2024-09-14T08:25:26.241 INFO:teuthology.orchestra.run.smithi135.stdout:5.14.0-505.el9.x86_64 2024-09-14T08:25:26.242 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-505.el9.x86_64 vs 5.14.0-505.el9.x86_64 2024-09-14T08:25:26.242 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-09-14T08:25:26.242 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-09-14T08:25:27.242 INFO:teuthology.task.kernel:Enabling kdb on cluster1.mon.a... 2024-09-14T08:25:27.243 DEBUG:teuthology.orchestra.run.smithi135:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-09-14T08:25:27.477 INFO:teuthology.orchestra.run.smithi135.stdout:ttyS1 2024-09-14T08:25:27.496 DEBUG:teuthology.parallel:result is None 2024-09-14T08:25:54.594 DEBUG:teuthology.orchestra.remote:timed out 2024-09-14T08:26:09.595 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi164.front.sepia.ceph.com' 2024-09-14T08:26:09.596 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi164.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-14T08:26:09.986 DEBUG:teuthology.orchestra.run.smithi164:> true 2024-09-14T08:26:10.336 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi164.front.sepia.ceph.com' 2024-09-14T08:26:10.336 INFO:teuthology.task.kernel:Checking kernel version of cluster2.mon.a, want "5.14.0-505.el9.x86_64"... 2024-09-14T08:26:10.337 DEBUG:teuthology.orchestra.run.smithi164:> uname -r 2024-09-14T08:26:10.354 INFO:teuthology.orchestra.run.smithi164.stdout:5.14.0-505.el9.x86_64 2024-09-14T08:26:10.354 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-505.el9.x86_64 vs 5.14.0-505.el9.x86_64 2024-09-14T08:26:10.354 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-09-14T08:26:10.354 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-09-14T08:26:11.355 INFO:teuthology.task.kernel:Enabling kdb on cluster2.mon.a... 2024-09-14T08:26:11.355 DEBUG:teuthology.orchestra.run.smithi164:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-09-14T08:26:11.422 INFO:teuthology.orchestra.run.smithi164.stdout:ttyS1 2024-09-14T08:26:11.446 DEBUG:teuthology.parallel:result is None 2024-09-14T08:26:11.446 INFO:teuthology.run_tasks:Running task internal.base... 2024-09-14T08:26:11.453 INFO:teuthology.task.internal:Creating test directory... 2024-09-14T08:26:11.454 DEBUG:teuthology.orchestra.run.smithi135:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-09-14T08:26:11.457 DEBUG:teuthology.orchestra.run.smithi164:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-09-14T08:26:11.474 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-09-14T08:26:11.530 INFO:teuthology.run_tasks:Running task internal.archive... 2024-09-14T08:26:11.540 INFO:teuthology.task.internal:Creating archive directory... 2024-09-14T08:26:11.540 DEBUG:teuthology.orchestra.run.smithi135:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-09-14T08:26:11.543 DEBUG:teuthology.orchestra.run.smithi164:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-09-14T08:26:11.572 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-09-14T08:26:11.579 INFO:teuthology.task.internal:Enabling coredump saving... 2024-09-14T08:26:11.579 DEBUG:teuthology.orchestra.run.smithi135:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-09-14T08:26:11.605 DEBUG:teuthology.orchestra.run.smithi164:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-09-14T08:26:11.645 INFO:teuthology.orchestra.run.smithi164.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-14T08:26:11.645 INFO:teuthology.orchestra.run.smithi135.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-14T08:26:11.657 INFO:teuthology.orchestra.run.smithi164.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-14T08:26:11.657 INFO:teuthology.orchestra.run.smithi135.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-14T08:26:11.659 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-09-14T08:26:11.666 INFO:teuthology.task.internal:Configuring sudo... 2024-09-14T08:26:11.666 DEBUG:teuthology.orchestra.run.smithi135:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-09-14T08:26:11.701 DEBUG:teuthology.orchestra.run.smithi164:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-09-14T08:26:11.730 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-09-14T08:26:11.739 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-09-14T08:26:11.739 DEBUG:teuthology.orchestra.run.smithi135:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-09-14T08:26:11.769 DEBUG:teuthology.orchestra.run.smithi164:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-09-14T08:26:11.787 DEBUG:teuthology.orchestra.run.smithi135:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-14T08:26:11.869 DEBUG:teuthology.orchestra.run.smithi135:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-14T08:26:11.921 DEBUG:teuthology.orchestra.run.smithi135:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-14T08:26:12.004 DEBUG:teuthology.orchestra.run.smithi135:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-14T08:26:12.044 DEBUG:teuthology.orchestra.run.smithi135:> set -ex 2024-09-14T08:26:12.044 DEBUG:teuthology.orchestra.run.smithi135:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-09-14T08:26:12.110 DEBUG:teuthology.orchestra.run.smithi164:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-14T08:26:12.154 DEBUG:teuthology.orchestra.run.smithi164:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-14T08:26:12.207 DEBUG:teuthology.orchestra.run.smithi164:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-14T08:26:12.291 DEBUG:teuthology.orchestra.run.smithi164:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-14T08:26:12.333 DEBUG:teuthology.orchestra.run.smithi164:> set -ex 2024-09-14T08:26:12.333 DEBUG:teuthology.orchestra.run.smithi164:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-09-14T08:26:12.399 DEBUG:teuthology.orchestra.run.smithi135:> sudo service rsyslog restart 2024-09-14T08:26:12.402 DEBUG:teuthology.orchestra.run.smithi164:> sudo service rsyslog restart 2024-09-14T08:26:12.442 INFO:teuthology.orchestra.run.smithi135.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-14T08:26:12.482 INFO:teuthology.orchestra.run.smithi164.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-14T08:26:12.998 INFO:teuthology.run_tasks:Running task internal.timer... 2024-09-14T08:26:13.006 INFO:teuthology.task.internal:Starting timer... 2024-09-14T08:26:13.006 INFO:teuthology.run_tasks:Running task pcp... 2024-09-14T08:26:13.026 INFO:teuthology.run_tasks:Running task selinux... 2024-09-14T08:26:13.036 DEBUG:teuthology.orchestra.run.smithi135:> sudo service auditd rotate 2024-09-14T08:26:13.090 INFO:teuthology.orchestra.run.smithi135.stdout:Rotating logs: 2024-09-14T08:26:13.091 DEBUG:teuthology.orchestra.run.smithi164:> sudo service auditd rotate 2024-09-14T08:26:13.161 INFO:teuthology.orchestra.run.smithi164.stdout:Rotating logs: 2024-09-14T08:26:13.163 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-09-14T08:26:13.163 DEBUG:teuthology.orchestra.run.smithi135:> /usr/sbin/getenforce 2024-09-14T08:26:13.194 INFO:teuthology.orchestra.run.smithi135.stdout:Permissive 2024-09-14T08:26:13.195 DEBUG:teuthology.orchestra.run.smithi164:> /usr/sbin/getenforce 2024-09-14T08:26:13.249 INFO:teuthology.orchestra.run.smithi164.stdout:Permissive 2024-09-14T08:26:13.249 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi135.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi164.front.sepia.ceph.com': 'permissive'} 2024-09-14T08:26:13.249 DEBUG:teuthology.orchestra.run.smithi135:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 2024-09-14T08:26:13.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T08:26:13.276 DEBUG:teuthology.orchestra.run.smithi164:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 2024-09-14T08:26:13.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T08:26:13.306 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-09-14T08:26:13.306 DEBUG:teuthology.orchestra.run.smithi135:> sudo /usr/sbin/setenforce permissive 2024-09-14T08:26:13.346 DEBUG:teuthology.orchestra.run.smithi164:> sudo /usr/sbin/setenforce permissive 2024-09-14T08:26:13.382 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-09-14T08:26:13.391 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-09-14T08:26:13.402 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-09-14T08:26:13.403 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi135.front.sepia.ceph.com,smithi164.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-09-14T08:29:58.658 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi135.front.sepia.ceph.com'), Remote(name='ubuntu@smithi164.front.sepia.ceph.com')] 2024-09-14T08:29:58.659 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi135.front.sepia.ceph.com' 2024-09-14T08:29:58.661 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi135.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-14T08:29:58.741 DEBUG:teuthology.orchestra.run.smithi135:> true 2024-09-14T08:29:58.828 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi135.front.sepia.ceph.com' 2024-09-14T08:29:58.828 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi164.front.sepia.ceph.com' 2024-09-14T08:29:58.829 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi164.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-14T08:29:58.909 DEBUG:teuthology.orchestra.run.smithi164:> true 2024-09-14T08:29:58.944 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi164.front.sepia.ceph.com' 2024-09-14T08:29:58.944 INFO:teuthology.run_tasks:Running task clock... 2024-09-14T08:29:58.954 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-09-14T08:29:58.954 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-09-14T08:29:58.955 DEBUG:teuthology.orchestra.run.smithi135:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-14T08:29:58.957 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-09-14T08:29:58.957 DEBUG:teuthology.orchestra.run.smithi164:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-14T08:29:58.985 INFO:teuthology.orchestra.run.smithi135.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-09-14T08:29:59.000 INFO:teuthology.orchestra.run.smithi135.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-09-14T08:29:59.011 INFO:teuthology.orchestra.run.smithi164.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-09-14T08:29:59.024 INFO:teuthology.orchestra.run.smithi135.stderr:sudo: ntpd: command not found 2024-09-14T08:29:59.025 INFO:teuthology.orchestra.run.smithi164.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-09-14T08:29:59.035 INFO:teuthology.orchestra.run.smithi135.stdout:506 Cannot talk to daemon 2024-09-14T08:29:59.049 INFO:teuthology.orchestra.run.smithi164.stderr:sudo: ntpd: command not found 2024-09-14T08:29:59.050 INFO:teuthology.orchestra.run.smithi135.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-09-14T08:29:59.060 INFO:teuthology.orchestra.run.smithi164.stdout:506 Cannot talk to daemon 2024-09-14T08:29:59.064 INFO:teuthology.orchestra.run.smithi135.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-09-14T08:29:59.074 INFO:teuthology.orchestra.run.smithi164.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-09-14T08:29:59.088 INFO:teuthology.orchestra.run.smithi164.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-09-14T08:29:59.117 INFO:teuthology.orchestra.run.smithi135.stderr:bash: line 1: ntpq: command not found 2024-09-14T08:29:59.120 INFO:teuthology.orchestra.run.smithi135.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-14T08:29:59.120 INFO:teuthology.orchestra.run.smithi135.stdout:=============================================================================== 2024-09-14T08:29:59.120 INFO:teuthology.orchestra.run.smithi135.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-14T08:29:59.120 INFO:teuthology.orchestra.run.smithi135.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-14T08:29:59.120 INFO:teuthology.orchestra.run.smithi135.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-14T08:29:59.120 INFO:teuthology.orchestra.run.smithi135.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-14T08:29:59.150 INFO:teuthology.orchestra.run.smithi164.stderr:bash: line 1: ntpq: command not found 2024-09-14T08:29:59.153 INFO:teuthology.orchestra.run.smithi164.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-14T08:29:59.154 INFO:teuthology.orchestra.run.smithi164.stdout:=============================================================================== 2024-09-14T08:29:59.154 INFO:teuthology.orchestra.run.smithi164.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-14T08:29:59.154 INFO:teuthology.orchestra.run.smithi164.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-14T08:29:59.154 INFO:teuthology.orchestra.run.smithi164.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-14T08:29:59.154 INFO:teuthology.orchestra.run.smithi164.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-14T08:29:59.155 INFO:teuthology.run_tasks:Running task install... 2024-09-14T08:29:59.163 DEBUG:teuthology.task.install:project ceph 2024-09-14T08:29:59.163 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': '7f24e589328ae9e639c0009d93340e5b235d0a47'}} 2024-09-14T08:29:59.163 DEBUG:teuthology.task.install:config {'extra_packages': ['rbd-mirror'], 'flavor': 'default', 'sha1': '7f24e589328ae9e639c0009d93340e5b235d0a47'} 2024-09-14T08:29:59.164 INFO:teuthology.task.install:Using flavor: default 2024-09-14T08:29:59.169 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'ceph-volume', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'ceph-volume', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2024-09-14T08:29:59.170 INFO:teuthology.task.install:extra packages: ['rbd-mirror'] 2024-09-14T08:29:59.170 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': None, 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': [], 'extra_packages': ['rbd-mirror'], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': '7f24e589328ae9e639c0009d93340e5b235d0a47', 'tag': None, 'wait_for_package': False} 2024-09-14T08:29:59.171 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=7f24e589328ae9e639c0009d93340e5b235d0a47 2024-09-14T08:29:59.173 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': None, 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': [], 'extra_packages': ['rbd-mirror'], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': '7f24e589328ae9e639c0009d93340e5b235d0a47', 'tag': None, 'wait_for_package': False} 2024-09-14T08:29:59.173 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=7f24e589328ae9e639c0009d93340e5b235d0a47 2024-09-14T08:29:59.313 INFO:teuthology.task.install.rpm:Pulling from https://1.chacra.ceph.com/r/ceph/main/7f24e589328ae9e639c0009d93340e5b235d0a47/centos/9/flavors/default/ 2024-09-14T08:29:59.314 INFO:teuthology.task.install.rpm:Package version is 19.3.0-4775.g7f24e589 2024-09-14T08:29:59.320 INFO:teuthology.task.install.rpm:Pulling from https://1.chacra.ceph.com/r/ceph/main/7f24e589328ae9e639c0009d93340e5b235d0a47/centos/9/flavors/default/ 2024-09-14T08:29:59.320 INFO:teuthology.task.install.rpm:Package version is 19.3.0-4775.g7f24e589 2024-09-14T08:29:59.425 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://1.chacra.ceph.com/r/ceph/main/7f24e589328ae9e639c0009d93340e5b235d0a47/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://1.chacra.ceph.com/r/ceph/main/7f24e589328ae9e639c0009d93340e5b235d0a47/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://1.chacra.ceph.com/r/ceph/main/7f24e589328ae9e639c0009d93340e5b235d0a47/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2024-09-14T08:29:59.425 DEBUG:teuthology.orchestra.run.smithi135:> set -ex 2024-09-14T08:29:59.426 DEBUG:teuthology.orchestra.run.smithi135:> sudo dd of=/etc/yum.repos.d/ceph.repo 2024-09-14T08:29:59.429 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://1.chacra.ceph.com/r/ceph/main/7f24e589328ae9e639c0009d93340e5b235d0a47/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://1.chacra.ceph.com/r/ceph/main/7f24e589328ae9e639c0009d93340e5b235d0a47/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://1.chacra.ceph.com/r/ceph/main/7f24e589328ae9e639c0009d93340e5b235d0a47/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2024-09-14T08:29:59.429 DEBUG:teuthology.orchestra.run.smithi164:> set -ex 2024-09-14T08:29:59.429 DEBUG:teuthology.orchestra.run.smithi164:> sudo dd of=/etc/yum.repos.d/ceph.repo 2024-09-14T08:29:59.451 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, ceph-volume, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd, rbd-mirror on remote rpm x86_64 2024-09-14T08:29:59.451 DEBUG:teuthology.orchestra.run.smithi135:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;sha1/7f24e589328ae9e639c0009d93340e5b235d0a47/;g' /etc/yum.repos.d/ceph.repo ; fi 2024-09-14T08:29:59.456 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, ceph-volume, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd, rbd-mirror on remote rpm x86_64 2024-09-14T08:29:59.457 DEBUG:teuthology.orchestra.run.smithi164:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;sha1/7f24e589328ae9e639c0009d93340e5b235d0a47/;g' /etc/yum.repos.d/ceph.repo ; fi 2024-09-14T08:29:59.518 DEBUG:teuthology.orchestra.run.smithi135:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2024-09-14T08:29:59.523 DEBUG:teuthology.orchestra.run.smithi164:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2024-09-14T08:29:59.595 DEBUG:teuthology.orchestra.run.smithi135:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2024-09-14T08:29:59.597 DEBUG:teuthology.orchestra.run.smithi164:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2024-09-14T08:29:59.659 INFO:teuthology.orchestra.run.smithi135.stdout:check_obsoletes = 1 2024-09-14T08:29:59.660 DEBUG:teuthology.orchestra.run.smithi135:> sudo yum clean all 2024-09-14T08:29:59.661 INFO:teuthology.orchestra.run.smithi164.stdout:check_obsoletes = 1 2024-09-14T08:29:59.663 DEBUG:teuthology.orchestra.run.smithi164:> sudo yum clean all 2024-09-14T08:29:59.891 INFO:teuthology.orchestra.run.smithi135.stdout:45 files removed 2024-09-14T08:29:59.900 INFO:teuthology.orchestra.run.smithi164.stdout:45 files removed 2024-09-14T08:29:59.919 DEBUG:teuthology.orchestra.run.smithi135:> sudo yum -y install ceph-radosgw 2024-09-14T08:29:59.927 DEBUG:teuthology.orchestra.run.smithi164:> sudo yum -y install ceph-radosgw 2024-09-14T08:30:00.481 INFO:teuthology.orchestra.run.smithi164.stdout:ceph packages for x86_64 316 kB/s | 84 kB 00:00 2024-09-14T08:30:00.482 INFO:teuthology.orchestra.run.smithi135.stdout:ceph packages for x86_64 306 kB/s | 84 kB 00:00 2024-09-14T08:30:00.773 INFO:teuthology.orchestra.run.smithi164.stdout:ceph noarch packages 97 kB/s | 24 kB 00:00 2024-09-14T08:30:00.774 INFO:teuthology.orchestra.run.smithi135.stdout:ceph noarch packages 97 kB/s | 24 kB 00:00 2024-09-14T08:30:01.049 INFO:teuthology.orchestra.run.smithi164.stdout:ceph source packages 8.1 kB/s | 1.9 kB 00:00 2024-09-14T08:30:01.050 INFO:teuthology.orchestra.run.smithi135.stdout:ceph source packages 8.1 kB/s | 1.9 kB 00:00 2024-09-14T08:30:01.524 INFO:teuthology.orchestra.run.smithi164.stdout:CentOS Stream 9 - BaseOS 19 MB/s | 8.2 MB 00:00 2024-09-14T08:30:02.824 INFO:teuthology.orchestra.run.smithi135.stdout:CentOS Stream 9 - BaseOS 4.7 MB/s | 8.2 MB 00:01 2024-09-14T08:30:03.442 INFO:teuthology.orchestra.run.smithi164.stdout:CentOS Stream 9 - AppStream 25 MB/s | 20 MB 00:00 2024-09-14T08:30:04.609 INFO:teuthology.orchestra.run.smithi135.stdout:CentOS Stream 9 - AppStream 30 MB/s | 20 MB 00:00 2024-09-14T08:30:07.902 INFO:teuthology.orchestra.run.smithi164.stdout:CentOS Stream 9 - CRB 9.7 MB/s | 6.5 MB 00:00 2024-09-14T08:30:09.360 INFO:teuthology.orchestra.run.smithi135.stdout:CentOS Stream 9 - CRB 10 MB/s | 6.5 MB 00:00 2024-09-14T08:30:09.594 INFO:teuthology.orchestra.run.smithi164.stdout:CentOS Stream 9 - Extras packages 109 kB/s | 19 kB 00:00 2024-09-14T08:30:10.932 INFO:teuthology.orchestra.run.smithi164.stdout:Extra Packages for Enterprise Linux 19 MB/s | 23 MB 00:01 2024-09-14T08:30:10.961 INFO:teuthology.orchestra.run.smithi135.stdout:CentOS Stream 9 - Extras packages 70 kB/s | 19 kB 00:00 2024-09-14T08:30:11.840 INFO:teuthology.orchestra.run.smithi135.stdout:Extra Packages for Enterprise Linux 30 MB/s | 23 MB 00:00 2024-09-14T08:30:16.421 INFO:teuthology.orchestra.run.smithi164.stdout:lab-extras 31 kB/s | 1.7 kB 00:00 2024-09-14T08:30:17.388 INFO:teuthology.orchestra.run.smithi135.stdout:lab-extras 30 kB/s | 1.7 kB 00:00 2024-09-14T08:30:18.806 INFO:teuthology.orchestra.run.smithi164.stdout:Dependencies resolved. 2024-09-14T08:30:18.829 INFO:teuthology.orchestra.run.smithi164.stdout:================================================================================ 2024-09-14T08:30:18.829 INFO:teuthology.orchestra.run.smithi164.stdout: Package Arch Version Repository Size 2024-09-14T08:30:18.829 INFO:teuthology.orchestra.run.smithi164.stdout:================================================================================ 2024-09-14T08:30:18.829 INFO:teuthology.orchestra.run.smithi164.stdout:Installing: 2024-09-14T08:30:18.829 INFO:teuthology.orchestra.run.smithi164.stdout: ceph-radosgw x86_64 2:19.3.0-4775.g7f24e589.el9 ceph 11 M 2024-09-14T08:30:18.829 INFO:teuthology.orchestra.run.smithi164.stdout:Upgrading: 2024-09-14T08:30:18.829 INFO:teuthology.orchestra.run.smithi164.stdout: librados2 x86_64 2:19.3.0-4775.g7f24e589.el9 ceph 3.5 M 2024-09-14T08:30:18.829 INFO:teuthology.orchestra.run.smithi164.stdout: librbd1 x86_64 2:19.3.0-4775.g7f24e589.el9 ceph 3.2 M 2024-09-14T08:30:18.829 INFO:teuthology.orchestra.run.smithi164.stdout:Installing dependencies: 2024-09-14T08:30:18.829 INFO:teuthology.orchestra.run.smithi164.stdout: boost-program-options x86_64 1.75.0-8.el9 appstream 107 k 2024-09-14T08:30:18.829 INFO:teuthology.orchestra.run.smithi164.stdout: ceph-base x86_64 2:19.3.0-4775.g7f24e589.el9 ceph 5.5 M 2024-09-14T08:30:18.830 INFO:teuthology.orchestra.run.smithi164.stdout: ceph-common x86_64 2:19.3.0-4775.g7f24e589.el9 ceph 22 M 2024-09-14T08:30:18.830 INFO:teuthology.orchestra.run.smithi164.stdout: ceph-selinux x86_64 2:19.3.0-4775.g7f24e589.el9 ceph 25 k 2024-09-14T08:30:18.830 INFO:teuthology.orchestra.run.smithi164.stdout: gperftools-libs x86_64 2.9.1-3.el9 epel 308 k 2024-09-14T08:30:18.830 INFO:teuthology.orchestra.run.smithi164.stdout: libarrow x86_64 9.0.0-11.el9 epel 4.4 M 2024-09-14T08:30:18.830 INFO:teuthology.orchestra.run.smithi164.stdout: libarrow-doc noarch 9.0.0-11.el9 epel 26 k 2024-09-14T08:30:18.830 INFO:teuthology.orchestra.run.smithi164.stdout: libcephfs2 x86_64 2:19.3.0-4775.g7f24e589.el9 ceph 755 k 2024-09-14T08:30:18.830 INFO:teuthology.orchestra.run.smithi164.stdout: libnbd x86_64 1.20.2-2.el9 appstream 171 k 2024-09-14T08:30:18.830 INFO:teuthology.orchestra.run.smithi164.stdout: liboath x86_64 2.6.7-2.el9 epel 49 k 2024-09-14T08:30:18.830 INFO:teuthology.orchestra.run.smithi164.stdout: libpmemobj x86_64 1.12.1-1.el9 appstream 160 k 2024-09-14T08:30:18.830 INFO:teuthology.orchestra.run.smithi164.stdout: librabbitmq x86_64 0.11.0-7.el9 appstream 45 k 2024-09-14T08:30:18.830 INFO:teuthology.orchestra.run.smithi164.stdout: libradosstriper1 x86_64 2:19.3.0-4775.g7f24e589.el9 ceph 508 k 2024-09-14T08:30:18.830 INFO:teuthology.orchestra.run.smithi164.stdout: librdkafka x86_64 1.6.1-102.el9 appstream 662 k 2024-09-14T08:30:18.830 INFO:teuthology.orchestra.run.smithi164.stdout: librgw2 x86_64 2:19.3.0-4775.g7f24e589.el9 ceph 5.5 M 2024-09-14T08:30:18.831 INFO:teuthology.orchestra.run.smithi164.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2024-09-14T08:30:18.831 INFO:teuthology.orchestra.run.smithi164.stdout: lttng-ust x86_64 2.12.0-6.el9 appstream 292 k 2024-09-14T08:30:18.831 INFO:teuthology.orchestra.run.smithi164.stdout: parquet-libs x86_64 9.0.0-11.el9 epel 839 k 2024-09-14T08:30:18.831 INFO:teuthology.orchestra.run.smithi164.stdout: python3-ceph-argparse x86_64 2:19.3.0-4775.g7f24e589.el9 ceph 45 k 2024-09-14T08:30:18.831 INFO:teuthology.orchestra.run.smithi164.stdout: python3-ceph-common x86_64 2:19.3.0-4775.g7f24e589.el9 ceph 143 k 2024-09-14T08:30:18.831 INFO:teuthology.orchestra.run.smithi164.stdout: python3-cephfs x86_64 2:19.3.0-4775.g7f24e589.el9 ceph 161 k 2024-09-14T08:30:18.831 INFO:teuthology.orchestra.run.smithi164.stdout: python3-prettytable noarch 0.7.2-27.el9 appstream 42 k 2024-09-14T08:30:18.831 INFO:teuthology.orchestra.run.smithi164.stdout: python3-rados x86_64 2:19.3.0-4775.g7f24e589.el9 ceph 321 k 2024-09-14T08:30:18.831 INFO:teuthology.orchestra.run.smithi164.stdout: python3-rbd x86_64 2:19.3.0-4775.g7f24e589.el9 ceph 306 k 2024-09-14T08:30:18.831 INFO:teuthology.orchestra.run.smithi164.stdout: python3-rgw x86_64 2:19.3.0-4775.g7f24e589.el9 ceph 99 k 2024-09-14T08:30:18.831 INFO:teuthology.orchestra.run.smithi164.stdout: qatlib x86_64 24.02.0-1.el9 appstream 221 k 2024-09-14T08:30:18.831 INFO:teuthology.orchestra.run.smithi164.stdout: qatzip-libs x86_64 1.2.0-1.el9 appstream 47 k 2024-09-14T08:30:18.831 INFO:teuthology.orchestra.run.smithi164.stdout: re2 x86_64 1:20211101-20.el9 epel 191 k 2024-09-14T08:30:18.831 INFO:teuthology.orchestra.run.smithi164.stdout: thrift x86_64 0.15.0-2.el9 epel 1.6 M 2024-09-14T08:30:18.832 INFO:teuthology.orchestra.run.smithi164.stdout:Installing weak dependencies: 2024-09-14T08:30:18.832 INFO:teuthology.orchestra.run.smithi164.stdout: qatlib-service x86_64 24.02.0-1.el9 appstream 36 k 2024-09-14T08:30:18.832 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:30:18.832 INFO:teuthology.orchestra.run.smithi164.stdout:Transaction Summary 2024-09-14T08:30:18.832 INFO:teuthology.orchestra.run.smithi164.stdout:================================================================================ 2024-09-14T08:30:18.832 INFO:teuthology.orchestra.run.smithi164.stdout:Install 31 Packages 2024-09-14T08:30:18.832 INFO:teuthology.orchestra.run.smithi164.stdout:Upgrade 2 Packages 2024-09-14T08:30:18.832 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:30:18.833 INFO:teuthology.orchestra.run.smithi164.stdout:Total download size: 62 M 2024-09-14T08:30:18.833 INFO:teuthology.orchestra.run.smithi164.stdout:Downloading Packages: 2024-09-14T08:30:19.800 INFO:teuthology.orchestra.run.smithi135.stdout:Dependencies resolved. 2024-09-14T08:30:19.815 INFO:teuthology.orchestra.run.smithi135.stdout:================================================================================ 2024-09-14T08:30:19.815 INFO:teuthology.orchestra.run.smithi135.stdout: Package Arch Version Repository Size 2024-09-14T08:30:19.815 INFO:teuthology.orchestra.run.smithi135.stdout:================================================================================ 2024-09-14T08:30:19.815 INFO:teuthology.orchestra.run.smithi135.stdout:Installing: 2024-09-14T08:30:19.816 INFO:teuthology.orchestra.run.smithi135.stdout: ceph-radosgw x86_64 2:19.3.0-4775.g7f24e589.el9 ceph 11 M 2024-09-14T08:30:19.816 INFO:teuthology.orchestra.run.smithi135.stdout:Upgrading: 2024-09-14T08:30:19.816 INFO:teuthology.orchestra.run.smithi135.stdout: librados2 x86_64 2:19.3.0-4775.g7f24e589.el9 ceph 3.5 M 2024-09-14T08:30:19.816 INFO:teuthology.orchestra.run.smithi135.stdout: librbd1 x86_64 2:19.3.0-4775.g7f24e589.el9 ceph 3.2 M 2024-09-14T08:30:19.816 INFO:teuthology.orchestra.run.smithi135.stdout:Installing dependencies: 2024-09-14T08:30:19.816 INFO:teuthology.orchestra.run.smithi135.stdout: boost-program-options x86_64 1.75.0-8.el9 appstream 107 k 2024-09-14T08:30:19.816 INFO:teuthology.orchestra.run.smithi135.stdout: ceph-base x86_64 2:19.3.0-4775.g7f24e589.el9 ceph 5.5 M 2024-09-14T08:30:19.816 INFO:teuthology.orchestra.run.smithi135.stdout: ceph-common x86_64 2:19.3.0-4775.g7f24e589.el9 ceph 22 M 2024-09-14T08:30:19.816 INFO:teuthology.orchestra.run.smithi135.stdout: ceph-selinux x86_64 2:19.3.0-4775.g7f24e589.el9 ceph 25 k 2024-09-14T08:30:19.816 INFO:teuthology.orchestra.run.smithi135.stdout: gperftools-libs x86_64 2.9.1-3.el9 epel 308 k 2024-09-14T08:30:19.816 INFO:teuthology.orchestra.run.smithi135.stdout: libarrow x86_64 9.0.0-11.el9 epel 4.4 M 2024-09-14T08:30:19.816 INFO:teuthology.orchestra.run.smithi135.stdout: libarrow-doc noarch 9.0.0-11.el9 epel 26 k 2024-09-14T08:30:19.817 INFO:teuthology.orchestra.run.smithi135.stdout: libcephfs2 x86_64 2:19.3.0-4775.g7f24e589.el9 ceph 755 k 2024-09-14T08:30:19.817 INFO:teuthology.orchestra.run.smithi135.stdout: libnbd x86_64 1.20.2-2.el9 appstream 171 k 2024-09-14T08:30:19.817 INFO:teuthology.orchestra.run.smithi135.stdout: liboath x86_64 2.6.7-2.el9 epel 49 k 2024-09-14T08:30:19.817 INFO:teuthology.orchestra.run.smithi135.stdout: libpmemobj x86_64 1.12.1-1.el9 appstream 160 k 2024-09-14T08:30:19.817 INFO:teuthology.orchestra.run.smithi135.stdout: librabbitmq x86_64 0.11.0-7.el9 appstream 45 k 2024-09-14T08:30:19.817 INFO:teuthology.orchestra.run.smithi135.stdout: libradosstriper1 x86_64 2:19.3.0-4775.g7f24e589.el9 ceph 508 k 2024-09-14T08:30:19.817 INFO:teuthology.orchestra.run.smithi135.stdout: librdkafka x86_64 1.6.1-102.el9 appstream 662 k 2024-09-14T08:30:19.817 INFO:teuthology.orchestra.run.smithi135.stdout: librgw2 x86_64 2:19.3.0-4775.g7f24e589.el9 ceph 5.5 M 2024-09-14T08:30:19.817 INFO:teuthology.orchestra.run.smithi135.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2024-09-14T08:30:19.817 INFO:teuthology.orchestra.run.smithi135.stdout: lttng-ust x86_64 2.12.0-6.el9 appstream 292 k 2024-09-14T08:30:19.817 INFO:teuthology.orchestra.run.smithi135.stdout: parquet-libs x86_64 9.0.0-11.el9 epel 839 k 2024-09-14T08:30:19.817 INFO:teuthology.orchestra.run.smithi135.stdout: python3-ceph-argparse x86_64 2:19.3.0-4775.g7f24e589.el9 ceph 45 k 2024-09-14T08:30:19.818 INFO:teuthology.orchestra.run.smithi135.stdout: python3-ceph-common x86_64 2:19.3.0-4775.g7f24e589.el9 ceph 143 k 2024-09-14T08:30:19.818 INFO:teuthology.orchestra.run.smithi135.stdout: python3-cephfs x86_64 2:19.3.0-4775.g7f24e589.el9 ceph 161 k 2024-09-14T08:30:19.818 INFO:teuthology.orchestra.run.smithi135.stdout: python3-prettytable noarch 0.7.2-27.el9 appstream 42 k 2024-09-14T08:30:19.818 INFO:teuthology.orchestra.run.smithi135.stdout: python3-rados x86_64 2:19.3.0-4775.g7f24e589.el9 ceph 321 k 2024-09-14T08:30:19.818 INFO:teuthology.orchestra.run.smithi135.stdout: python3-rbd x86_64 2:19.3.0-4775.g7f24e589.el9 ceph 306 k 2024-09-14T08:30:19.818 INFO:teuthology.orchestra.run.smithi135.stdout: python3-rgw x86_64 2:19.3.0-4775.g7f24e589.el9 ceph 99 k 2024-09-14T08:30:19.818 INFO:teuthology.orchestra.run.smithi135.stdout: qatlib x86_64 24.02.0-1.el9 appstream 221 k 2024-09-14T08:30:19.818 INFO:teuthology.orchestra.run.smithi135.stdout: qatzip-libs x86_64 1.2.0-1.el9 appstream 47 k 2024-09-14T08:30:19.818 INFO:teuthology.orchestra.run.smithi135.stdout: re2 x86_64 1:20211101-20.el9 epel 191 k 2024-09-14T08:30:19.818 INFO:teuthology.orchestra.run.smithi135.stdout: thrift x86_64 0.15.0-2.el9 epel 1.6 M 2024-09-14T08:30:19.818 INFO:teuthology.orchestra.run.smithi135.stdout:Installing weak dependencies: 2024-09-14T08:30:19.818 INFO:teuthology.orchestra.run.smithi135.stdout: qatlib-service x86_64 24.02.0-1.el9 appstream 36 k 2024-09-14T08:30:19.818 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:30:19.819 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction Summary 2024-09-14T08:30:19.819 INFO:teuthology.orchestra.run.smithi135.stdout:================================================================================ 2024-09-14T08:30:19.819 INFO:teuthology.orchestra.run.smithi135.stdout:Install 31 Packages 2024-09-14T08:30:19.819 INFO:teuthology.orchestra.run.smithi135.stdout:Upgrade 2 Packages 2024-09-14T08:30:19.819 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:30:19.819 INFO:teuthology.orchestra.run.smithi135.stdout:Total download size: 62 M 2024-09-14T08:30:19.820 INFO:teuthology.orchestra.run.smithi135.stdout:Downloading Packages: 2024-09-14T08:30:20.214 INFO:teuthology.orchestra.run.smithi164.stdout:(1/33): ceph-base-19.3.0-4775.g7f24e589.el9.x86 4.9 MB/s | 5.5 MB 00:01 2024-09-14T08:30:20.256 INFO:teuthology.orchestra.run.smithi164.stdout:(2/33): ceph-selinux-19.3.0-4775.g7f24e589.el9. 606 kB/s | 25 kB 00:00 2024-09-14T08:30:20.406 INFO:teuthology.orchestra.run.smithi164.stdout:(3/33): ceph-radosgw-19.3.0-4775.g7f24e589.el9. 8.3 MB/s | 11 MB 00:01 2024-09-14T08:30:20.440 INFO:teuthology.orchestra.run.smithi164.stdout:(4/33): libcephfs2-19.3.0-4775.g7f24e589.el9.x8 4.0 MB/s | 755 kB 00:00 2024-09-14T08:30:20.473 INFO:teuthology.orchestra.run.smithi164.stdout:(5/33): libradosstriper1-19.3.0-4775.g7f24e589. 7.4 MB/s | 508 kB 00:00 2024-09-14T08:30:20.540 INFO:teuthology.orchestra.run.smithi164.stdout:(6/33): python3-ceph-argparse-19.3.0-4775.g7f24 678 kB/s | 45 kB 00:00 2024-09-14T08:30:20.615 INFO:teuthology.orchestra.run.smithi164.stdout:(7/33): python3-ceph-common-19.3.0-4775.g7f24e5 1.9 MB/s | 143 kB 00:00 2024-09-14T08:30:20.707 INFO:teuthology.orchestra.run.smithi164.stdout:(8/33): python3-cephfs-19.3.0-4775.g7f24e589.el 1.7 MB/s | 161 kB 00:00 2024-09-14T08:30:20.791 INFO:teuthology.orchestra.run.smithi164.stdout:(9/33): python3-rados-19.3.0-4775.g7f24e589.el9 3.8 MB/s | 321 kB 00:00 2024-09-14T08:30:20.900 INFO:teuthology.orchestra.run.smithi164.stdout:(10/33): python3-rbd-19.3.0-4775.g7f24e589.el9. 2.8 MB/s | 306 kB 00:00 2024-09-14T08:30:21.122 INFO:teuthology.orchestra.run.smithi135.stdout:(1/33): ceph-base-19.3.0-4775.g7f24e589.el9.x86 6.3 MB/s | 5.5 MB 00:00 2024-09-14T08:30:21.175 INFO:teuthology.orchestra.run.smithi164.stdout:(11/33): ceph-common-19.3.0-4775.g7f24e589.el9. 10 MB/s | 22 MB 00:02 2024-09-14T08:30:21.200 INFO:teuthology.orchestra.run.smithi164.stdout:(12/33): python3-rgw-19.3.0-4775.g7f24e589.el9. 331 kB/s | 99 kB 00:00 2024-09-14T08:30:21.276 INFO:teuthology.orchestra.run.smithi164.stdout:(13/33): librgw2-19.3.0-4775.g7f24e589.el9.x86_ 6.6 MB/s | 5.5 MB 00:00 2024-09-14T08:30:21.285 INFO:teuthology.orchestra.run.smithi135.stdout:(2/33): ceph-radosgw-19.3.0-4775.g7f24e589.el9. 11 MB/s | 11 MB 00:01 2024-09-14T08:30:21.306 INFO:teuthology.orchestra.run.smithi135.stdout:(3/33): ceph-selinux-19.3.0-4775.g7f24e589.el9. 137 kB/s | 25 kB 00:00 2024-09-14T08:30:21.339 INFO:teuthology.orchestra.run.smithi135.stdout:(4/33): libcephfs2-19.3.0-4775.g7f24e589.el9.x8 13 MB/s | 755 kB 00:00 2024-09-14T08:30:21.373 INFO:teuthology.orchestra.run.smithi135.stdout:(5/33): libradosstriper1-19.3.0-4775.g7f24e589. 7.4 MB/s | 508 kB 00:00 2024-09-14T08:30:21.448 INFO:teuthology.orchestra.run.smithi135.stdout:(6/33): python3-ceph-argparse-19.3.0-4775.g7f24 603 kB/s | 45 kB 00:00 2024-09-14T08:30:21.532 INFO:teuthology.orchestra.run.smithi135.stdout:(7/33): python3-ceph-common-19.3.0-4775.g7f24e5 1.7 MB/s | 143 kB 00:00 2024-09-14T08:30:21.668 INFO:teuthology.orchestra.run.smithi164.stdout:(14/33): boost-program-options-1.75.0-8.el9.x86 217 kB/s | 107 kB 00:00 2024-09-14T08:30:21.674 INFO:teuthology.orchestra.run.smithi135.stdout:(8/33): librgw2-19.3.0-4775.g7f24e589.el9.x86_6 17 MB/s | 5.5 MB 00:00 2024-09-14T08:30:21.699 INFO:teuthology.orchestra.run.smithi135.stdout:(9/33): python3-cephfs-19.3.0-4775.g7f24e589.el 967 kB/s | 161 kB 00:00 2024-09-14T08:30:21.733 INFO:teuthology.orchestra.run.smithi135.stdout:(10/33): python3-rados-19.3.0-4775.g7f24e589.el 5.4 MB/s | 321 kB 00:00 2024-09-14T08:30:21.751 INFO:teuthology.orchestra.run.smithi164.stdout:(15/33): libnbd-1.20.2-2.el9.x86_64.rpm 310 kB/s | 171 kB 00:00 2024-09-14T08:30:21.768 INFO:teuthology.orchestra.run.smithi164.stdout:(16/33): librabbitmq-0.11.0-7.el9.x86_64.rpm 451 kB/s | 45 kB 00:00 2024-09-14T08:30:21.775 INFO:teuthology.orchestra.run.smithi135.stdout:(11/33): python3-rbd-19.3.0-4775.g7f24e589.el9. 4.0 MB/s | 306 kB 00:00 2024-09-14T08:30:21.800 INFO:teuthology.orchestra.run.smithi135.stdout:(12/33): python3-rgw-19.3.0-4775.g7f24e589.el9. 1.4 MB/s | 99 kB 00:00 2024-09-14T08:30:21.818 INFO:teuthology.orchestra.run.smithi164.stdout:(17/33): libpmemobj-1.12.1-1.el9.x86_64.rpm 295 kB/s | 160 kB 00:00 2024-09-14T08:30:21.927 INFO:teuthology.orchestra.run.smithi164.stdout:(18/33): python3-prettytable-0.7.2-27.el9.noarc 390 kB/s | 42 kB 00:00 2024-09-14T08:30:22.009 INFO:teuthology.orchestra.run.smithi135.stdout:(13/33): ceph-common-19.3.0-4775.g7f24e589.el9. 12 MB/s | 22 MB 00:01 2024-09-14T08:30:22.060 INFO:teuthology.orchestra.run.smithi164.stdout:(19/33): lttng-ust-2.12.0-6.el9.x86_64.rpm 1.0 MB/s | 292 kB 00:00 2024-09-14T08:30:22.067 INFO:teuthology.orchestra.run.smithi135.stdout:(14/33): boost-program-options-1.75.0-8.el9.x86 366 kB/s | 107 kB 00:00 2024-09-14T08:30:22.110 INFO:teuthology.orchestra.run.smithi135.stdout:(15/33): libnbd-1.20.2-2.el9.x86_64.rpm 552 kB/s | 171 kB 00:00 2024-09-14T08:30:22.127 INFO:teuthology.orchestra.run.smithi164.stdout:(20/33): qatlib-24.02.0-1.el9.x86_64.rpm 1.1 MB/s | 221 kB 00:00 2024-09-14T08:30:22.134 INFO:teuthology.orchestra.run.smithi135.stdout:(16/33): librabbitmq-0.11.0-7.el9.x86_64.rpm 677 kB/s | 45 kB 00:00 2024-09-14T08:30:22.168 INFO:teuthology.orchestra.run.smithi135.stdout:(17/33): libpmemobj-1.12.1-1.el9.x86_64.rpm 1.0 MB/s | 160 kB 00:00 2024-09-14T08:30:22.169 INFO:teuthology.orchestra.run.smithi164.stdout:(21/33): qatlib-service-24.02.0-1.el9.x86_64.rp 330 kB/s | 36 kB 00:00 2024-09-14T08:30:22.193 INFO:teuthology.orchestra.run.smithi135.stdout:(18/33): lttng-ust-2.12.0-6.el9.x86_64.rpm 4.9 MB/s | 292 kB 00:00 2024-09-14T08:30:22.219 INFO:teuthology.orchestra.run.smithi135.stdout:(19/33): python3-prettytable-0.7.2-27.el9.noarc 840 kB/s | 42 kB 00:00 2024-09-14T08:30:22.228 INFO:teuthology.orchestra.run.smithi164.stdout:(22/33): librdkafka-1.6.1-102.el9.x86_64.rpm 1.4 MB/s | 662 kB 00:00 2024-09-14T08:30:22.244 INFO:teuthology.orchestra.run.smithi135.stdout:(20/33): qatlib-24.02.0-1.el9.x86_64.rpm 4.3 MB/s | 221 kB 00:00 2024-09-14T08:30:22.253 INFO:teuthology.orchestra.run.smithi164.stdout:(23/33): qatzip-libs-1.2.0-1.el9.x86_64.rpm 377 kB/s | 47 kB 00:00 2024-09-14T08:30:22.269 INFO:teuthology.orchestra.run.smithi135.stdout:(21/33): librdkafka-1.6.1-102.el9.x86_64.rpm 4.0 MB/s | 662 kB 00:00 2024-09-14T08:30:22.295 INFO:teuthology.orchestra.run.smithi135.stdout:(22/33): qatlib-service-24.02.0-1.el9.x86_64.rp 475 kB/s | 36 kB 00:00 2024-09-14T08:30:22.295 INFO:teuthology.orchestra.run.smithi164.stdout:(24/33): gperftools-libs-2.9.1-3.el9.x86_64.rpm 2.4 MB/s | 308 kB 00:00 2024-09-14T08:30:22.320 INFO:teuthology.orchestra.run.smithi135.stdout:(23/33): qatzip-libs-1.2.0-1.el9.x86_64.rpm 628 kB/s | 47 kB 00:00 2024-09-14T08:30:22.320 INFO:teuthology.orchestra.run.smithi164.stdout:(25/33): liboath-2.6.7-2.el9.x86_64.rpm 2.1 MB/s | 49 kB 00:00 2024-09-14T08:30:22.346 INFO:teuthology.orchestra.run.smithi164.stdout:(26/33): libarrow-doc-9.0.0-11.el9.noarch.rpm 307 kB/s | 26 kB 00:00 2024-09-14T08:30:22.378 INFO:teuthology.orchestra.run.smithi135.stdout:(24/33): gperftools-libs-2.9.1-3.el9.x86_64.rpm 2.8 MB/s | 308 kB 00:00 2024-09-14T08:30:22.379 INFO:teuthology.orchestra.run.smithi164.stdout:(27/33): libunwind-1.6.2-1.el9.x86_64.rpm 1.1 MB/s | 67 kB 00:00 2024-09-14T08:30:22.412 INFO:teuthology.orchestra.run.smithi135.stdout:(25/33): liboath-2.6.7-2.el9.x86_64.rpm 1.5 MB/s | 49 kB 00:00 2024-09-14T08:30:22.413 INFO:teuthology.orchestra.run.smithi164.stdout:(28/33): re2-20211101-20.el9.x86_64.rpm 5.6 MB/s | 191 kB 00:00 2024-09-14T08:30:22.438 INFO:teuthology.orchestra.run.smithi135.stdout:(26/33): libarrow-doc-9.0.0-11.el9.noarch.rpm 253 kB/s | 26 kB 00:00 2024-09-14T08:30:22.455 INFO:teuthology.orchestra.run.smithi164.stdout:(29/33): parquet-libs-9.0.0-11.el9.x86_64.rpm 7.5 MB/s | 839 kB 00:00 2024-09-14T08:30:22.463 INFO:teuthology.orchestra.run.smithi135.stdout:(27/33): libunwind-1.6.2-1.el9.x86_64.rpm 1.3 MB/s | 67 kB 00:00 2024-09-14T08:30:22.514 INFO:teuthology.orchestra.run.smithi164.stdout:(30/33): thrift-0.15.0-2.el9.x86_64.rpm 16 MB/s | 1.6 MB 00:00 2024-09-14T08:30:22.580 INFO:teuthology.orchestra.run.smithi135.stdout:(28/33): re2-20211101-20.el9.x86_64.rpm 1.6 MB/s | 191 kB 00:00 2024-09-14T08:30:22.589 INFO:teuthology.orchestra.run.smithi164.stdout:(31/33): libarrow-9.0.0-11.el9.x86_64.rpm 12 MB/s | 4.4 MB 00:00 2024-09-14T08:30:22.638 INFO:teuthology.orchestra.run.smithi135.stdout:(29/33): parquet-libs-9.0.0-11.el9.x86_64.rpm 4.1 MB/s | 839 kB 00:00 2024-09-14T08:30:22.681 INFO:teuthology.orchestra.run.smithi135.stdout:(30/33): thrift-0.15.0-2.el9.x86_64.rpm 16 MB/s | 1.6 MB 00:00 2024-09-14T08:30:22.740 INFO:teuthology.orchestra.run.smithi164.stdout:(32/33): librbd1-19.3.0-4775.g7f24e589.el9.x86_ 14 MB/s | 3.2 MB 00:00 2024-09-14T08:30:22.756 INFO:teuthology.orchestra.run.smithi135.stdout:(31/33): libarrow-9.0.0-11.el9.x86_64.rpm 9.6 MB/s | 4.4 MB 00:00 2024-09-14T08:30:22.798 INFO:teuthology.orchestra.run.smithi164.stdout:(33/33): librados2-19.3.0-4775.g7f24e589.el9.x8 10 MB/s | 3.5 MB 00:00 2024-09-14T08:30:22.803 INFO:teuthology.orchestra.run.smithi164.stdout:-------------------------------------------------------------------------------- 2024-09-14T08:30:22.803 INFO:teuthology.orchestra.run.smithi164.stdout:Total 16 MB/s | 62 MB 00:03 2024-09-14T08:30:22.889 INFO:teuthology.orchestra.run.smithi135.stdout:(32/33): librbd1-19.3.0-4775.g7f24e589.el9.x86_ 16 MB/s | 3.2 MB 00:00 2024-09-14T08:30:22.919 INFO:teuthology.orchestra.run.smithi164.stdout:Running transaction check 2024-09-14T08:30:22.957 INFO:teuthology.orchestra.run.smithi164.stdout:Transaction check succeeded. 2024-09-14T08:30:22.957 INFO:teuthology.orchestra.run.smithi164.stdout:Running transaction test 2024-09-14T08:30:22.981 INFO:teuthology.orchestra.run.smithi135.stdout:(33/33): librados2-19.3.0-4775.g7f24e589.el9.x8 10 MB/s | 3.5 MB 00:00 2024-09-14T08:30:22.986 INFO:teuthology.orchestra.run.smithi135.stdout:-------------------------------------------------------------------------------- 2024-09-14T08:30:22.986 INFO:teuthology.orchestra.run.smithi135.stdout:Total 20 MB/s | 62 MB 00:03 2024-09-14T08:30:23.102 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction check 2024-09-14T08:30:23.138 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction check succeeded. 2024-09-14T08:30:23.138 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction test 2024-09-14T08:30:23.644 INFO:teuthology.orchestra.run.smithi164.stdout:Transaction test succeeded. 2024-09-14T08:30:23.645 INFO:teuthology.orchestra.run.smithi164.stdout:Running transaction 2024-09-14T08:30:23.827 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction test succeeded. 2024-09-14T08:30:23.828 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction 2024-09-14T08:30:25.197 INFO:teuthology.orchestra.run.smithi164.stdout: Preparing : 1/1 2024-09-14T08:30:25.282 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 1/35 2024-09-14T08:30:25.320 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 2/35 2024-09-14T08:30:25.353 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 3/35 2024-09-14T08:30:25.364 INFO:teuthology.orchestra.run.smithi135.stdout: Preparing : 1/1 2024-09-14T08:30:25.376 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : qatlib-service-24.02.0-1.el9.x86_64 4/35 2024-09-14T08:30:25.400 INFO:teuthology.orchestra.run.smithi164.stdout: Running scriptlet: qatlib-service-24.02.0-1.el9.x86_64 4/35 2024-09-14T08:30:25.462 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 1/35 2024-09-14T08:30:25.495 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 2/35 2024-09-14T08:30:25.528 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 3/35 2024-09-14T08:30:25.559 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : qatlib-service-24.02.0-1.el9.x86_64 4/35 2024-09-14T08:30:25.587 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: qatlib-service-24.02.0-1.el9.x86_64 4/35 2024-09-14T08:30:25.596 INFO:teuthology.orchestra.run.smithi164.stdout: Running scriptlet: qatlib-24.02.0-1.el9.x86_64 5/35 2024-09-14T08:30:25.649 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : qatlib-24.02.0-1.el9.x86_64 5/35 2024-09-14T08:30:25.787 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: qatlib-24.02.0-1.el9.x86_64 5/35 2024-09-14T08:30:25.839 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : qatlib-24.02.0-1.el9.x86_64 5/35 2024-09-14T08:30:26.050 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : thrift-0.15.0-2.el9.x86_64 6/35 2024-09-14T08:30:26.105 INFO:teuthology.orchestra.run.smithi164.stdout: Upgrading : librados2-2:19.3.0-4775.g7f24e589.el9.x86_64 7/35 2024-09-14T08:30:26.198 INFO:teuthology.orchestra.run.smithi164.stdout: Running scriptlet: librados2-2:19.3.0-4775.g7f24e589.el9.x86_64 7/35 2024-09-14T08:30:26.238 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : thrift-0.15.0-2.el9.x86_64 6/35 2024-09-14T08:30:26.288 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : python3-rados-2:19.3.0-4775.g7f24e589.el9.x86_64 8/35 2024-09-14T08:30:26.288 INFO:teuthology.orchestra.run.smithi135.stdout: Upgrading : librados2-2:19.3.0-4775.g7f24e589.el9.x86_64 7/35 2024-09-14T08:30:26.315 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : libcephfs2-2:19.3.0-4775.g7f24e589.el9.x86_64 9/35 2024-09-14T08:30:26.381 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: librados2-2:19.3.0-4775.g7f24e589.el9.x86_64 7/35 2024-09-14T08:30:26.387 INFO:teuthology.orchestra.run.smithi164.stdout: Running scriptlet: libcephfs2-2:19.3.0-4775.g7f24e589.el9.x86_64 9/35 2024-09-14T08:30:26.432 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : liboath-2.6.7-2.el9.x86_64 10/35 2024-09-14T08:30:26.485 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : python3-ceph-argparse-2:19.3.0-4775.g7f24e589.el9. 11/35 2024-09-14T08:30:26.489 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-rados-2:19.3.0-4775.g7f24e589.el9.x86_64 8/35 2024-09-14T08:30:26.523 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : libcephfs2-2:19.3.0-4775.g7f24e589.el9.x86_64 9/35 2024-09-14T08:30:26.555 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : python3-cephfs-2:19.3.0-4775.g7f24e589.el9.x86_64 12/35 2024-09-14T08:30:26.586 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: libcephfs2-2:19.3.0-4775.g7f24e589.el9.x86_64 9/35 2024-09-14T08:30:26.592 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : libradosstriper1-2:19.3.0-4775.g7f24e589.el9.x86_6 13/35 2024-09-14T08:30:26.632 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : liboath-2.6.7-2.el9.x86_64 10/35 2024-09-14T08:30:26.657 INFO:teuthology.orchestra.run.smithi164.stdout: Running scriptlet: libradosstriper1-2:19.3.0-4775.g7f24e589.el9.x86_6 13/35 2024-09-14T08:30:26.676 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-ceph-argparse-2:19.3.0-4775.g7f24e589.el9. 11/35 2024-09-14T08:30:26.699 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : qatzip-libs-1.2.0-1.el9.x86_64 14/35 2024-09-14T08:30:26.744 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-cephfs-2:19.3.0-4775.g7f24e589.el9.x86_64 12/35 2024-09-14T08:30:26.767 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : libradosstriper1-2:19.3.0-4775.g7f24e589.el9.x86_6 13/35 2024-09-14T08:30:26.811 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : re2-1:20211101-20.el9.x86_64 15/35 2024-09-14T08:30:26.831 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: libradosstriper1-2:19.3.0-4775.g7f24e589.el9.x86_6 13/35 2024-09-14T08:30:26.850 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : libarrow-9.0.0-11.el9.x86_64 16/35 2024-09-14T08:30:26.866 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : qatzip-libs-1.2.0-1.el9.x86_64 14/35 2024-09-14T08:30:26.908 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 17/35 2024-09-14T08:30:26.957 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : gperftools-libs-2.9.1-3.el9.x86_64 18/35 2024-09-14T08:30:26.974 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : re2-1:20211101-20.el9.x86_64 15/35 2024-09-14T08:30:26.986 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : libarrow-doc-9.0.0-11.el9.noarch 19/35 2024-09-14T08:30:27.009 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : libarrow-9.0.0-11.el9.x86_64 16/35 2024-09-14T08:30:27.020 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : python3-prettytable-0.7.2-27.el9.noarch 20/35 2024-09-14T08:30:27.057 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 21/35 2024-09-14T08:30:27.058 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 17/35 2024-09-14T08:30:27.099 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : gperftools-libs-2.9.1-3.el9.x86_64 18/35 2024-09-14T08:30:27.127 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : libarrow-doc-9.0.0-11.el9.noarch 19/35 2024-09-14T08:30:27.168 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-prettytable-0.7.2-27.el9.noarch 20/35 2024-09-14T08:30:27.208 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 21/35 2024-09-14T08:30:27.379 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : libnbd-1.20.2-2.el9.x86_64 22/35 2024-09-14T08:30:27.425 INFO:teuthology.orchestra.run.smithi164.stdout: Upgrading : librbd1-2:19.3.0-4775.g7f24e589.el9.x86_64 23/35 2024-09-14T08:30:27.523 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : libnbd-1.20.2-2.el9.x86_64 22/35 2024-09-14T08:30:27.524 INFO:teuthology.orchestra.run.smithi164.stdout: Running scriptlet: librbd1-2:19.3.0-4775.g7f24e589.el9.x86_64 23/35 2024-09-14T08:30:27.564 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : python3-rbd-2:19.3.0-4775.g7f24e589.el9.x86_64 24/35 2024-09-14T08:30:27.574 INFO:teuthology.orchestra.run.smithi135.stdout: Upgrading : librbd1-2:19.3.0-4775.g7f24e589.el9.x86_64 23/35 2024-09-14T08:30:27.616 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : boost-program-options-1.75.0-8.el9.x86_64 25/35 2024-09-14T08:30:27.819 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: librbd1-2:19.3.0-4775.g7f24e589.el9.x86_64 23/35 2024-09-14T08:30:27.923 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-rbd-2:19.3.0-4775.g7f24e589.el9.x86_64 24/35 2024-09-14T08:30:27.965 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : boost-program-options-1.75.0-8.el9.x86_64 25/35 2024-09-14T08:30:28.119 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : parquet-libs-9.0.0-11.el9.x86_64 26/35 2024-09-14T08:30:28.153 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : librgw2-2:19.3.0-4775.g7f24e589.el9.x86_64 27/35 2024-09-14T08:30:28.224 INFO:teuthology.orchestra.run.smithi164.stdout: Running scriptlet: librgw2-2:19.3.0-4775.g7f24e589.el9.x86_64 27/35 2024-09-14T08:30:28.304 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : python3-rgw-2:19.3.0-4775.g7f24e589.el9.x86_64 28/35 2024-09-14T08:30:28.330 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : python3-ceph-common-2:19.3.0-4775.g7f24e589.el9.x8 29/35 2024-09-14T08:30:28.460 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : parquet-libs-9.0.0-11.el9.x86_64 26/35 2024-09-14T08:30:28.493 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : librgw2-2:19.3.0-4775.g7f24e589.el9.x86_64 27/35 2024-09-14T08:30:28.574 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: librgw2-2:19.3.0-4775.g7f24e589.el9.x86_64 27/35 2024-09-14T08:30:28.653 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-rgw-2:19.3.0-4775.g7f24e589.el9.x86_64 28/35 2024-09-14T08:30:28.687 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-ceph-common-2:19.3.0-4775.g7f24e589.el9.x8 29/35 2024-09-14T08:30:30.699 INFO:teuthology.orchestra.run.smithi164.stdout: Running scriptlet: ceph-common-2:19.3.0-4775.g7f24e589.el9.x86_64 30/35 2024-09-14T08:30:30.735 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : ceph-common-2:19.3.0-4775.g7f24e589.el9.x86_64 30/35 2024-09-14T08:30:31.067 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: ceph-common-2:19.3.0-4775.g7f24e589.el9.x86_64 30/35 2024-09-14T08:30:31.109 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : ceph-common-2:19.3.0-4775.g7f24e589.el9.x86_64 30/35 2024-09-14T08:30:31.314 INFO:teuthology.orchestra.run.smithi164.stdout: Running scriptlet: ceph-common-2:19.3.0-4775.g7f24e589.el9.x86_64 30/35 2024-09-14T08:30:31.355 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : ceph-base-2:19.3.0-4775.g7f24e589.el9.x86_64 31/35 2024-09-14T08:30:31.447 INFO:teuthology.orchestra.run.smithi164.stdout: Running scriptlet: ceph-base-2:19.3.0-4775.g7f24e589.el9.x86_64 31/35 2024-09-14T08:30:31.447 INFO:teuthology.orchestra.run.smithi164.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2024-09-14T08:30:31.447 INFO:teuthology.orchestra.run.smithi164.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2024-09-14T08:30:31.447 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:30:31.490 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : ceph-selinux-2:19.3.0-4775.g7f24e589.el9.x86_64 32/35 2024-09-14T08:30:31.693 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: ceph-common-2:19.3.0-4775.g7f24e589.el9.x86_64 30/35 2024-09-14T08:30:31.729 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : ceph-base-2:19.3.0-4775.g7f24e589.el9.x86_64 31/35 2024-09-14T08:30:31.846 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: ceph-base-2:19.3.0-4775.g7f24e589.el9.x86_64 31/35 2024-09-14T08:30:31.846 INFO:teuthology.orchestra.run.smithi135.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2024-09-14T08:30:31.846 INFO:teuthology.orchestra.run.smithi135.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2024-09-14T08:30:31.847 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:30:31.880 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : ceph-selinux-2:19.3.0-4775.g7f24e589.el9.x86_64 32/35 2024-09-14T08:30:41.531 INFO:teuthology.orchestra.run.smithi164.stdout: Running scriptlet: ceph-selinux-2:19.3.0-4775.g7f24e589.el9.x86_64 32/35 2024-09-14T08:30:41.531 INFO:teuthology.orchestra.run.smithi164.stdout:skipping the directory /sys 2024-09-14T08:30:41.531 INFO:teuthology.orchestra.run.smithi164.stdout:skipping the directory /proc 2024-09-14T08:30:41.531 INFO:teuthology.orchestra.run.smithi164.stdout:skipping the directory /mnt 2024-09-14T08:30:41.531 INFO:teuthology.orchestra.run.smithi164.stdout:skipping the directory /var/tmp 2024-09-14T08:30:41.532 INFO:teuthology.orchestra.run.smithi164.stdout:skipping the directory /home 2024-09-14T08:30:41.532 INFO:teuthology.orchestra.run.smithi164.stdout:skipping the directory /root 2024-09-14T08:30:41.532 INFO:teuthology.orchestra.run.smithi164.stdout:skipping the directory /tmp 2024-09-14T08:30:41.532 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:30:41.928 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: ceph-selinux-2:19.3.0-4775.g7f24e589.el9.x86_64 32/35 2024-09-14T08:30:41.928 INFO:teuthology.orchestra.run.smithi135.stdout:skipping the directory /sys 2024-09-14T08:30:41.928 INFO:teuthology.orchestra.run.smithi135.stdout:skipping the directory /proc 2024-09-14T08:30:41.929 INFO:teuthology.orchestra.run.smithi135.stdout:skipping the directory /mnt 2024-09-14T08:30:41.929 INFO:teuthology.orchestra.run.smithi135.stdout:skipping the directory /var/tmp 2024-09-14T08:30:41.929 INFO:teuthology.orchestra.run.smithi135.stdout:skipping the directory /home 2024-09-14T08:30:41.929 INFO:teuthology.orchestra.run.smithi135.stdout:skipping the directory /root 2024-09-14T08:30:41.929 INFO:teuthology.orchestra.run.smithi135.stdout:skipping the directory /tmp 2024-09-14T08:30:41.929 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:30:42.593 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : ceph-radosgw-2:19.3.0-4775.g7f24e589.el9.x86_64 33/35 2024-09-14T08:30:42.619 INFO:teuthology.orchestra.run.smithi164.stdout: Running scriptlet: ceph-radosgw-2:19.3.0-4775.g7f24e589.el9.x86_64 33/35 2024-09-14T08:30:42.619 INFO:teuthology.orchestra.run.smithi164.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-14T08:30:42.619 INFO:teuthology.orchestra.run.smithi164.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2024-09-14T08:30:42.619 INFO:teuthology.orchestra.run.smithi164.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-09-14T08:30:42.619 INFO:teuthology.orchestra.run.smithi164.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-09-14T08:30:42.620 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:30:42.621 INFO:teuthology.orchestra.run.smithi164.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 34/35 2024-09-14T08:30:42.704 INFO:teuthology.orchestra.run.smithi164.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 34/35 2024-09-14T08:30:42.705 INFO:teuthology.orchestra.run.smithi164.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 35/35 2024-09-14T08:30:42.981 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : ceph-radosgw-2:19.3.0-4775.g7f24e589.el9.x86_64 33/35 2024-09-14T08:30:43.006 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: ceph-radosgw-2:19.3.0-4775.g7f24e589.el9.x86_64 33/35 2024-09-14T08:30:43.007 INFO:teuthology.orchestra.run.smithi135.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-14T08:30:43.007 INFO:teuthology.orchestra.run.smithi135.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2024-09-14T08:30:43.007 INFO:teuthology.orchestra.run.smithi135.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-09-14T08:30:43.007 INFO:teuthology.orchestra.run.smithi135.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-09-14T08:30:43.007 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:30:43.008 INFO:teuthology.orchestra.run.smithi135.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 34/35 2024-09-14T08:30:43.092 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 34/35 2024-09-14T08:30:43.092 INFO:teuthology.orchestra.run.smithi135.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 35/35 2024-09-14T08:30:43.921 INFO:teuthology.orchestra.run.smithi164.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 35/35 2024-09-14T08:30:43.922 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : ceph-base-2:19.3.0-4775.g7f24e589.el9.x86_64 1/35 2024-09-14T08:30:43.922 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : ceph-common-2:19.3.0-4775.g7f24e589.el9.x86_64 2/35 2024-09-14T08:30:43.922 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : ceph-radosgw-2:19.3.0-4775.g7f24e589.el9.x86_64 3/35 2024-09-14T08:30:43.922 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : ceph-selinux-2:19.3.0-4775.g7f24e589.el9.x86_64 4/35 2024-09-14T08:30:43.922 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : libcephfs2-2:19.3.0-4775.g7f24e589.el9.x86_64 5/35 2024-09-14T08:30:43.922 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : libradosstriper1-2:19.3.0-4775.g7f24e589.el9.x86_6 6/35 2024-09-14T08:30:43.922 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : librgw2-2:19.3.0-4775.g7f24e589.el9.x86_64 7/35 2024-09-14T08:30:43.922 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : python3-ceph-argparse-2:19.3.0-4775.g7f24e589.el9. 8/35 2024-09-14T08:30:43.922 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : python3-ceph-common-2:19.3.0-4775.g7f24e589.el9.x8 9/35 2024-09-14T08:30:43.922 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : python3-cephfs-2:19.3.0-4775.g7f24e589.el9.x86_64 10/35 2024-09-14T08:30:43.922 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : python3-rados-2:19.3.0-4775.g7f24e589.el9.x86_64 11/35 2024-09-14T08:30:43.922 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : python3-rbd-2:19.3.0-4775.g7f24e589.el9.x86_64 12/35 2024-09-14T08:30:43.923 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : python3-rgw-2:19.3.0-4775.g7f24e589.el9.x86_64 13/35 2024-09-14T08:30:43.923 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : boost-program-options-1.75.0-8.el9.x86_64 14/35 2024-09-14T08:30:43.923 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : libnbd-1.20.2-2.el9.x86_64 15/35 2024-09-14T08:30:43.923 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 16/35 2024-09-14T08:30:43.923 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 17/35 2024-09-14T08:30:43.923 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 18/35 2024-09-14T08:30:43.923 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 19/35 2024-09-14T08:30:43.923 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : python3-prettytable-0.7.2-27.el9.noarch 20/35 2024-09-14T08:30:43.923 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : qatlib-24.02.0-1.el9.x86_64 21/35 2024-09-14T08:30:43.923 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : qatlib-service-24.02.0-1.el9.x86_64 22/35 2024-09-14T08:30:43.923 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : qatzip-libs-1.2.0-1.el9.x86_64 23/35 2024-09-14T08:30:43.923 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : gperftools-libs-2.9.1-3.el9.x86_64 24/35 2024-09-14T08:30:43.923 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : libarrow-9.0.0-11.el9.x86_64 25/35 2024-09-14T08:30:43.923 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : libarrow-doc-9.0.0-11.el9.noarch 26/35 2024-09-14T08:30:43.924 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : liboath-2.6.7-2.el9.x86_64 27/35 2024-09-14T08:30:43.924 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 28/35 2024-09-14T08:30:43.924 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : parquet-libs-9.0.0-11.el9.x86_64 29/35 2024-09-14T08:30:43.924 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : re2-1:20211101-20.el9.x86_64 30/35 2024-09-14T08:30:43.924 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : thrift-0.15.0-2.el9.x86_64 31/35 2024-09-14T08:30:43.924 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : librados2-2:19.3.0-4775.g7f24e589.el9.x86_64 32/35 2024-09-14T08:30:43.924 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 33/35 2024-09-14T08:30:43.924 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : librbd1-2:19.3.0-4775.g7f24e589.el9.x86_64 34/35 2024-09-14T08:30:44.428 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 35/35 2024-09-14T08:30:44.428 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : ceph-base-2:19.3.0-4775.g7f24e589.el9.x86_64 1/35 2024-09-14T08:30:44.428 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : ceph-common-2:19.3.0-4775.g7f24e589.el9.x86_64 2/35 2024-09-14T08:30:44.428 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : ceph-radosgw-2:19.3.0-4775.g7f24e589.el9.x86_64 3/35 2024-09-14T08:30:44.428 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : ceph-selinux-2:19.3.0-4775.g7f24e589.el9.x86_64 4/35 2024-09-14T08:30:44.428 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : libcephfs2-2:19.3.0-4775.g7f24e589.el9.x86_64 5/35 2024-09-14T08:30:44.428 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : libradosstriper1-2:19.3.0-4775.g7f24e589.el9.x86_6 6/35 2024-09-14T08:30:44.428 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : librgw2-2:19.3.0-4775.g7f24e589.el9.x86_64 7/35 2024-09-14T08:30:44.429 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-ceph-argparse-2:19.3.0-4775.g7f24e589.el9. 8/35 2024-09-14T08:30:44.429 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-ceph-common-2:19.3.0-4775.g7f24e589.el9.x8 9/35 2024-09-14T08:30:44.429 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-cephfs-2:19.3.0-4775.g7f24e589.el9.x86_64 10/35 2024-09-14T08:30:44.429 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-rados-2:19.3.0-4775.g7f24e589.el9.x86_64 11/35 2024-09-14T08:30:44.429 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-rbd-2:19.3.0-4775.g7f24e589.el9.x86_64 12/35 2024-09-14T08:30:44.429 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-rgw-2:19.3.0-4775.g7f24e589.el9.x86_64 13/35 2024-09-14T08:30:44.429 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : boost-program-options-1.75.0-8.el9.x86_64 14/35 2024-09-14T08:30:44.429 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : libnbd-1.20.2-2.el9.x86_64 15/35 2024-09-14T08:30:44.429 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 16/35 2024-09-14T08:30:44.429 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 17/35 2024-09-14T08:30:44.429 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 18/35 2024-09-14T08:30:44.429 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 19/35 2024-09-14T08:30:44.429 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-prettytable-0.7.2-27.el9.noarch 20/35 2024-09-14T08:30:44.429 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : qatlib-24.02.0-1.el9.x86_64 21/35 2024-09-14T08:30:44.430 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : qatlib-service-24.02.0-1.el9.x86_64 22/35 2024-09-14T08:30:44.430 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : qatzip-libs-1.2.0-1.el9.x86_64 23/35 2024-09-14T08:30:44.430 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : gperftools-libs-2.9.1-3.el9.x86_64 24/35 2024-09-14T08:30:44.430 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : libarrow-9.0.0-11.el9.x86_64 25/35 2024-09-14T08:30:44.430 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : libarrow-doc-9.0.0-11.el9.noarch 26/35 2024-09-14T08:30:44.430 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : liboath-2.6.7-2.el9.x86_64 27/35 2024-09-14T08:30:44.431 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 28/35 2024-09-14T08:30:44.431 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : parquet-libs-9.0.0-11.el9.x86_64 29/35 2024-09-14T08:30:44.431 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : re2-1:20211101-20.el9.x86_64 30/35 2024-09-14T08:30:44.431 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : thrift-0.15.0-2.el9.x86_64 31/35 2024-09-14T08:30:44.431 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : librados2-2:19.3.0-4775.g7f24e589.el9.x86_64 32/35 2024-09-14T08:30:44.431 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 33/35 2024-09-14T08:30:44.431 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : librbd1-2:19.3.0-4775.g7f24e589.el9.x86_64 34/35 2024-09-14T08:30:44.820 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 35/35 2024-09-14T08:30:44.821 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:30:44.821 INFO:teuthology.orchestra.run.smithi164.stdout:Upgraded: 2024-09-14T08:30:44.821 INFO:teuthology.orchestra.run.smithi164.stdout: librados2-2:19.3.0-4775.g7f24e589.el9.x86_64 2024-09-14T08:30:44.821 INFO:teuthology.orchestra.run.smithi164.stdout: librbd1-2:19.3.0-4775.g7f24e589.el9.x86_64 2024-09-14T08:30:44.821 INFO:teuthology.orchestra.run.smithi164.stdout:Installed: 2024-09-14T08:30:44.821 INFO:teuthology.orchestra.run.smithi164.stdout: boost-program-options-1.75.0-8.el9.x86_64 2024-09-14T08:30:44.821 INFO:teuthology.orchestra.run.smithi164.stdout: ceph-base-2:19.3.0-4775.g7f24e589.el9.x86_64 2024-09-14T08:30:44.821 INFO:teuthology.orchestra.run.smithi164.stdout: ceph-common-2:19.3.0-4775.g7f24e589.el9.x86_64 2024-09-14T08:30:44.821 INFO:teuthology.orchestra.run.smithi164.stdout: ceph-radosgw-2:19.3.0-4775.g7f24e589.el9.x86_64 2024-09-14T08:30:44.821 INFO:teuthology.orchestra.run.smithi164.stdout: ceph-selinux-2:19.3.0-4775.g7f24e589.el9.x86_64 2024-09-14T08:30:44.821 INFO:teuthology.orchestra.run.smithi164.stdout: gperftools-libs-2.9.1-3.el9.x86_64 2024-09-14T08:30:44.822 INFO:teuthology.orchestra.run.smithi164.stdout: libarrow-9.0.0-11.el9.x86_64 2024-09-14T08:30:44.822 INFO:teuthology.orchestra.run.smithi164.stdout: libarrow-doc-9.0.0-11.el9.noarch 2024-09-14T08:30:44.822 INFO:teuthology.orchestra.run.smithi164.stdout: libcephfs2-2:19.3.0-4775.g7f24e589.el9.x86_64 2024-09-14T08:30:44.822 INFO:teuthology.orchestra.run.smithi164.stdout: libnbd-1.20.2-2.el9.x86_64 2024-09-14T08:30:44.822 INFO:teuthology.orchestra.run.smithi164.stdout: liboath-2.6.7-2.el9.x86_64 2024-09-14T08:30:44.822 INFO:teuthology.orchestra.run.smithi164.stdout: libpmemobj-1.12.1-1.el9.x86_64 2024-09-14T08:30:44.822 INFO:teuthology.orchestra.run.smithi164.stdout: librabbitmq-0.11.0-7.el9.x86_64 2024-09-14T08:30:44.822 INFO:teuthology.orchestra.run.smithi164.stdout: libradosstriper1-2:19.3.0-4775.g7f24e589.el9.x86_64 2024-09-14T08:30:44.822 INFO:teuthology.orchestra.run.smithi164.stdout: librdkafka-1.6.1-102.el9.x86_64 2024-09-14T08:30:44.822 INFO:teuthology.orchestra.run.smithi164.stdout: librgw2-2:19.3.0-4775.g7f24e589.el9.x86_64 2024-09-14T08:30:44.822 INFO:teuthology.orchestra.run.smithi164.stdout: libunwind-1.6.2-1.el9.x86_64 2024-09-14T08:30:44.822 INFO:teuthology.orchestra.run.smithi164.stdout: lttng-ust-2.12.0-6.el9.x86_64 2024-09-14T08:30:44.822 INFO:teuthology.orchestra.run.smithi164.stdout: parquet-libs-9.0.0-11.el9.x86_64 2024-09-14T08:30:44.822 INFO:teuthology.orchestra.run.smithi164.stdout: python3-ceph-argparse-2:19.3.0-4775.g7f24e589.el9.x86_64 2024-09-14T08:30:44.823 INFO:teuthology.orchestra.run.smithi164.stdout: python3-ceph-common-2:19.3.0-4775.g7f24e589.el9.x86_64 2024-09-14T08:30:44.823 INFO:teuthology.orchestra.run.smithi164.stdout: python3-cephfs-2:19.3.0-4775.g7f24e589.el9.x86_64 2024-09-14T08:30:44.823 INFO:teuthology.orchestra.run.smithi164.stdout: python3-prettytable-0.7.2-27.el9.noarch 2024-09-14T08:30:44.823 INFO:teuthology.orchestra.run.smithi164.stdout: python3-rados-2:19.3.0-4775.g7f24e589.el9.x86_64 2024-09-14T08:30:44.823 INFO:teuthology.orchestra.run.smithi164.stdout: python3-rbd-2:19.3.0-4775.g7f24e589.el9.x86_64 2024-09-14T08:30:44.823 INFO:teuthology.orchestra.run.smithi164.stdout: python3-rgw-2:19.3.0-4775.g7f24e589.el9.x86_64 2024-09-14T08:30:44.823 INFO:teuthology.orchestra.run.smithi164.stdout: qatlib-24.02.0-1.el9.x86_64 2024-09-14T08:30:44.823 INFO:teuthology.orchestra.run.smithi164.stdout: qatlib-service-24.02.0-1.el9.x86_64 2024-09-14T08:30:44.823 INFO:teuthology.orchestra.run.smithi164.stdout: qatzip-libs-1.2.0-1.el9.x86_64 2024-09-14T08:30:44.823 INFO:teuthology.orchestra.run.smithi164.stdout: re2-1:20211101-20.el9.x86_64 2024-09-14T08:30:44.823 INFO:teuthology.orchestra.run.smithi164.stdout: thrift-0.15.0-2.el9.x86_64 2024-09-14T08:30:44.823 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:30:44.823 INFO:teuthology.orchestra.run.smithi164.stdout:Complete! 2024-09-14T08:30:45.185 DEBUG:teuthology.orchestra.run.smithi164:> sudo yum -y install ceph-test 2024-09-14T08:30:45.323 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 35/35 2024-09-14T08:30:45.323 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:30:45.323 INFO:teuthology.orchestra.run.smithi135.stdout:Upgraded: 2024-09-14T08:30:45.323 INFO:teuthology.orchestra.run.smithi135.stdout: librados2-2:19.3.0-4775.g7f24e589.el9.x86_64 2024-09-14T08:30:45.323 INFO:teuthology.orchestra.run.smithi135.stdout: librbd1-2:19.3.0-4775.g7f24e589.el9.x86_64 2024-09-14T08:30:45.323 INFO:teuthology.orchestra.run.smithi135.stdout:Installed: 2024-09-14T08:30:45.323 INFO:teuthology.orchestra.run.smithi135.stdout: boost-program-options-1.75.0-8.el9.x86_64 2024-09-14T08:30:45.324 INFO:teuthology.orchestra.run.smithi135.stdout: ceph-base-2:19.3.0-4775.g7f24e589.el9.x86_64 2024-09-14T08:30:45.324 INFO:teuthology.orchestra.run.smithi135.stdout: ceph-common-2:19.3.0-4775.g7f24e589.el9.x86_64 2024-09-14T08:30:45.324 INFO:teuthology.orchestra.run.smithi135.stdout: ceph-radosgw-2:19.3.0-4775.g7f24e589.el9.x86_64 2024-09-14T08:30:45.324 INFO:teuthology.orchestra.run.smithi135.stdout: ceph-selinux-2:19.3.0-4775.g7f24e589.el9.x86_64 2024-09-14T08:30:45.324 INFO:teuthology.orchestra.run.smithi135.stdout: gperftools-libs-2.9.1-3.el9.x86_64 2024-09-14T08:30:45.324 INFO:teuthology.orchestra.run.smithi135.stdout: libarrow-9.0.0-11.el9.x86_64 2024-09-14T08:30:45.324 INFO:teuthology.orchestra.run.smithi135.stdout: libarrow-doc-9.0.0-11.el9.noarch 2024-09-14T08:30:45.324 INFO:teuthology.orchestra.run.smithi135.stdout: libcephfs2-2:19.3.0-4775.g7f24e589.el9.x86_64 2024-09-14T08:30:45.324 INFO:teuthology.orchestra.run.smithi135.stdout: libnbd-1.20.2-2.el9.x86_64 2024-09-14T08:30:45.324 INFO:teuthology.orchestra.run.smithi135.stdout: liboath-2.6.7-2.el9.x86_64 2024-09-14T08:30:45.324 INFO:teuthology.orchestra.run.smithi135.stdout: libpmemobj-1.12.1-1.el9.x86_64 2024-09-14T08:30:45.324 INFO:teuthology.orchestra.run.smithi135.stdout: librabbitmq-0.11.0-7.el9.x86_64 2024-09-14T08:30:45.324 INFO:teuthology.orchestra.run.smithi135.stdout: libradosstriper1-2:19.3.0-4775.g7f24e589.el9.x86_64 2024-09-14T08:30:45.325 INFO:teuthology.orchestra.run.smithi135.stdout: librdkafka-1.6.1-102.el9.x86_64 2024-09-14T08:30:45.325 INFO:teuthology.orchestra.run.smithi135.stdout: librgw2-2:19.3.0-4775.g7f24e589.el9.x86_64 2024-09-14T08:30:45.325 INFO:teuthology.orchestra.run.smithi135.stdout: libunwind-1.6.2-1.el9.x86_64 2024-09-14T08:30:45.325 INFO:teuthology.orchestra.run.smithi135.stdout: lttng-ust-2.12.0-6.el9.x86_64 2024-09-14T08:30:45.325 INFO:teuthology.orchestra.run.smithi135.stdout: parquet-libs-9.0.0-11.el9.x86_64 2024-09-14T08:30:45.325 INFO:teuthology.orchestra.run.smithi135.stdout: python3-ceph-argparse-2:19.3.0-4775.g7f24e589.el9.x86_64 2024-09-14T08:30:45.325 INFO:teuthology.orchestra.run.smithi135.stdout: python3-ceph-common-2:19.3.0-4775.g7f24e589.el9.x86_64 2024-09-14T08:30:45.325 INFO:teuthology.orchestra.run.smithi135.stdout: python3-cephfs-2:19.3.0-4775.g7f24e589.el9.x86_64 2024-09-14T08:30:45.325 INFO:teuthology.orchestra.run.smithi135.stdout: python3-prettytable-0.7.2-27.el9.noarch 2024-09-14T08:30:45.325 INFO:teuthology.orchestra.run.smithi135.stdout: python3-rados-2:19.3.0-4775.g7f24e589.el9.x86_64 2024-09-14T08:30:45.325 INFO:teuthology.orchestra.run.smithi135.stdout: python3-rbd-2:19.3.0-4775.g7f24e589.el9.x86_64 2024-09-14T08:30:45.325 INFO:teuthology.orchestra.run.smithi135.stdout: python3-rgw-2:19.3.0-4775.g7f24e589.el9.x86_64 2024-09-14T08:30:45.325 INFO:teuthology.orchestra.run.smithi135.stdout: qatlib-24.02.0-1.el9.x86_64 2024-09-14T08:30:45.326 INFO:teuthology.orchestra.run.smithi135.stdout: qatlib-service-24.02.0-1.el9.x86_64 2024-09-14T08:30:45.326 INFO:teuthology.orchestra.run.smithi135.stdout: qatzip-libs-1.2.0-1.el9.x86_64 2024-09-14T08:30:45.326 INFO:teuthology.orchestra.run.smithi135.stdout: re2-1:20211101-20.el9.x86_64 2024-09-14T08:30:45.326 INFO:teuthology.orchestra.run.smithi135.stdout: thrift-0.15.0-2.el9.x86_64 2024-09-14T08:30:45.326 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:30:45.326 INFO:teuthology.orchestra.run.smithi135.stdout:Complete! 2024-09-14T08:30:45.703 DEBUG:teuthology.orchestra.run.smithi135:> sudo yum -y install ceph-test 2024-09-14T08:30:45.761 INFO:teuthology.orchestra.run.smithi164.stdout:Last metadata expiration check: 0:00:29 ago on Sat 14 Sep 2024 08:30:16 AM UTC. 2024-09-14T08:30:45.893 INFO:teuthology.orchestra.run.smithi164.stdout:Dependencies resolved. 2024-09-14T08:30:45.893 INFO:teuthology.orchestra.run.smithi164.stdout:================================================================================ 2024-09-14T08:30:45.893 INFO:teuthology.orchestra.run.smithi164.stdout: Package Arch Version Repository Size 2024-09-14T08:30:45.893 INFO:teuthology.orchestra.run.smithi164.stdout:================================================================================ 2024-09-14T08:30:45.893 INFO:teuthology.orchestra.run.smithi164.stdout:Installing: 2024-09-14T08:30:45.893 INFO:teuthology.orchestra.run.smithi164.stdout: ceph-test x86_64 2:19.3.0-4775.g7f24e589.el9 ceph 50 M 2024-09-14T08:30:45.894 INFO:teuthology.orchestra.run.smithi164.stdout:Installing dependencies: 2024-09-14T08:30:45.894 INFO:teuthology.orchestra.run.smithi164.stdout: libcephsqlite x86_64 2:19.3.0-4775.g7f24e589.el9 ceph 162 k 2024-09-14T08:30:45.894 INFO:teuthology.orchestra.run.smithi164.stdout: socat x86_64 1.7.4.1-6.el9 appstream 304 k 2024-09-14T08:30:45.894 INFO:teuthology.orchestra.run.smithi164.stdout: xmlstarlet x86_64 1.6.1-20.el9 appstream 64 k 2024-09-14T08:30:45.894 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:30:45.894 INFO:teuthology.orchestra.run.smithi164.stdout:Transaction Summary 2024-09-14T08:30:45.894 INFO:teuthology.orchestra.run.smithi164.stdout:================================================================================ 2024-09-14T08:30:45.894 INFO:teuthology.orchestra.run.smithi164.stdout:Install 4 Packages 2024-09-14T08:30:45.894 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:30:45.895 INFO:teuthology.orchestra.run.smithi164.stdout:Total download size: 51 M 2024-09-14T08:30:45.895 INFO:teuthology.orchestra.run.smithi164.stdout:Installed size: 217 M 2024-09-14T08:30:45.895 INFO:teuthology.orchestra.run.smithi164.stdout:Downloading Packages: 2024-09-14T08:30:46.143 INFO:teuthology.orchestra.run.smithi164.stdout:(1/4): socat-1.7.4.1-6.el9.x86_64.rpm 1.5 MB/s | 304 kB 00:00 2024-09-14T08:30:46.169 INFO:teuthology.orchestra.run.smithi164.stdout:(2/4): libcephsqlite-19.3.0-4775.g7f24e589.el9. 722 kB/s | 162 kB 00:00 2024-09-14T08:30:46.194 INFO:teuthology.orchestra.run.smithi164.stdout:(3/4): xmlstarlet-1.6.1-20.el9.x86_64.rpm 1.2 MB/s | 64 kB 00:00 2024-09-14T08:30:46.279 INFO:teuthology.orchestra.run.smithi135.stdout:Last metadata expiration check: 0:00:29 ago on Sat 14 Sep 2024 08:30:17 AM UTC. 2024-09-14T08:30:46.424 INFO:teuthology.orchestra.run.smithi135.stdout:Dependencies resolved. 2024-09-14T08:30:46.424 INFO:teuthology.orchestra.run.smithi135.stdout:================================================================================ 2024-09-14T08:30:46.424 INFO:teuthology.orchestra.run.smithi135.stdout: Package Arch Version Repository Size 2024-09-14T08:30:46.424 INFO:teuthology.orchestra.run.smithi135.stdout:================================================================================ 2024-09-14T08:30:46.425 INFO:teuthology.orchestra.run.smithi135.stdout:Installing: 2024-09-14T08:30:46.425 INFO:teuthology.orchestra.run.smithi135.stdout: ceph-test x86_64 2:19.3.0-4775.g7f24e589.el9 ceph 50 M 2024-09-14T08:30:46.425 INFO:teuthology.orchestra.run.smithi135.stdout:Installing dependencies: 2024-09-14T08:30:46.425 INFO:teuthology.orchestra.run.smithi135.stdout: libcephsqlite x86_64 2:19.3.0-4775.g7f24e589.el9 ceph 162 k 2024-09-14T08:30:46.425 INFO:teuthology.orchestra.run.smithi135.stdout: socat x86_64 1.7.4.1-6.el9 appstream 304 k 2024-09-14T08:30:46.425 INFO:teuthology.orchestra.run.smithi135.stdout: xmlstarlet x86_64 1.6.1-20.el9 appstream 64 k 2024-09-14T08:30:46.425 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:30:46.425 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction Summary 2024-09-14T08:30:46.425 INFO:teuthology.orchestra.run.smithi135.stdout:================================================================================ 2024-09-14T08:30:46.425 INFO:teuthology.orchestra.run.smithi135.stdout:Install 4 Packages 2024-09-14T08:30:46.425 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:30:46.426 INFO:teuthology.orchestra.run.smithi135.stdout:Total download size: 51 M 2024-09-14T08:30:46.426 INFO:teuthology.orchestra.run.smithi135.stdout:Installed size: 217 M 2024-09-14T08:30:46.426 INFO:teuthology.orchestra.run.smithi135.stdout:Downloading Packages: 2024-09-14T08:30:46.714 INFO:teuthology.orchestra.run.smithi135.stdout:(1/4): socat-1.7.4.1-6.el9.x86_64.rpm 1.6 MB/s | 304 kB 00:00 2024-09-14T08:30:46.756 INFO:teuthology.orchestra.run.smithi135.stdout:(2/4): xmlstarlet-1.6.1-20.el9.x86_64.rpm 1.5 MB/s | 64 kB 00:00 2024-09-14T08:30:46.781 INFO:teuthology.orchestra.run.smithi135.stdout:(3/4): libcephsqlite-19.3.0-4775.g7f24e589.el9. 652 kB/s | 162 kB 00:00 2024-09-14T08:30:47.678 INFO:teuthology.orchestra.run.smithi164.stdout:(4/4): ceph-test-19.3.0-4775.g7f24e589.el9.x86_ 29 MB/s | 50 MB 00:01 2024-09-14T08:30:47.679 INFO:teuthology.orchestra.run.smithi164.stdout:-------------------------------------------------------------------------------- 2024-09-14T08:30:47.680 INFO:teuthology.orchestra.run.smithi164.stdout:Total 28 MB/s | 51 MB 00:01 2024-09-14T08:30:47.708 INFO:teuthology.orchestra.run.smithi164.stdout:Running transaction check 2024-09-14T08:30:47.728 INFO:teuthology.orchestra.run.smithi164.stdout:Transaction check succeeded. 2024-09-14T08:30:47.728 INFO:teuthology.orchestra.run.smithi164.stdout:Running transaction test 2024-09-14T08:30:48.129 INFO:teuthology.orchestra.run.smithi164.stdout:Transaction test succeeded. 2024-09-14T08:30:48.129 INFO:teuthology.orchestra.run.smithi164.stdout:Running transaction 2024-09-14T08:30:48.658 INFO:teuthology.orchestra.run.smithi164.stdout: Preparing : 1/1 2024-09-14T08:30:48.761 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 1/4 2024-09-14T08:30:48.773 INFO:teuthology.orchestra.run.smithi135.stdout:(4/4): ceph-test-19.3.0-4775.g7f24e589.el9.x86_ 22 MB/s | 50 MB 00:02 2024-09-14T08:30:48.775 INFO:teuthology.orchestra.run.smithi135.stdout:-------------------------------------------------------------------------------- 2024-09-14T08:30:48.776 INFO:teuthology.orchestra.run.smithi135.stdout:Total 22 MB/s | 51 MB 00:02 2024-09-14T08:30:48.801 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : socat-1.7.4.1-6.el9.x86_64 2/4 2024-09-14T08:30:48.804 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction check 2024-09-14T08:30:48.824 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction check succeeded. 2024-09-14T08:30:48.824 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction test 2024-09-14T08:30:48.870 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : libcephsqlite-2:19.3.0-4775.g7f24e589.el9.x86_64 3/4 2024-09-14T08:30:49.221 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction test succeeded. 2024-09-14T08:30:49.221 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction 2024-09-14T08:30:49.765 INFO:teuthology.orchestra.run.smithi135.stdout: Preparing : 1/1 2024-09-14T08:30:49.863 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 1/4 2024-09-14T08:30:49.906 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : socat-1.7.4.1-6.el9.x86_64 2/4 2024-09-14T08:30:49.933 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : libcephsqlite-2:19.3.0-4775.g7f24e589.el9.x86_64 3/4 2024-09-14T08:30:53.646 INFO:teuthology.orchestra.run.smithi164.stdout: Running scriptlet: libcephsqlite-2:19.3.0-4775.g7f24e589.el9.x86_64 3/4 2024-09-14T08:30:53.728 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : ceph-test-2:19.3.0-4775.g7f24e589.el9.x86_64 4/4 2024-09-14T08:30:54.698 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: libcephsqlite-2:19.3.0-4775.g7f24e589.el9.x86_64 3/4 2024-09-14T08:30:54.782 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : ceph-test-2:19.3.0-4775.g7f24e589.el9.x86_64 4/4 2024-09-14T08:30:56.259 INFO:teuthology.orchestra.run.smithi164.stdout: Running scriptlet: ceph-test-2:19.3.0-4775.g7f24e589.el9.x86_64 4/4 2024-09-14T08:30:56.260 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : ceph-test-2:19.3.0-4775.g7f24e589.el9.x86_64 1/4 2024-09-14T08:30:56.260 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : libcephsqlite-2:19.3.0-4775.g7f24e589.el9.x86_64 2/4 2024-09-14T08:30:56.260 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : socat-1.7.4.1-6.el9.x86_64 3/4 2024-09-14T08:30:56.593 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 4/4 2024-09-14T08:30:56.593 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:30:56.593 INFO:teuthology.orchestra.run.smithi164.stdout:Installed: 2024-09-14T08:30:56.593 INFO:teuthology.orchestra.run.smithi164.stdout: ceph-test-2:19.3.0-4775.g7f24e589.el9.x86_64 2024-09-14T08:30:56.593 INFO:teuthology.orchestra.run.smithi164.stdout: libcephsqlite-2:19.3.0-4775.g7f24e589.el9.x86_64 2024-09-14T08:30:56.593 INFO:teuthology.orchestra.run.smithi164.stdout: socat-1.7.4.1-6.el9.x86_64 2024-09-14T08:30:56.593 INFO:teuthology.orchestra.run.smithi164.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2024-09-14T08:30:56.593 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:30:56.594 INFO:teuthology.orchestra.run.smithi164.stdout:Complete! 2024-09-14T08:30:56.869 DEBUG:teuthology.orchestra.run.smithi164:> sudo yum -y install ceph 2024-09-14T08:30:57.249 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: ceph-test-2:19.3.0-4775.g7f24e589.el9.x86_64 4/4 2024-09-14T08:30:57.250 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : ceph-test-2:19.3.0-4775.g7f24e589.el9.x86_64 1/4 2024-09-14T08:30:57.250 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : libcephsqlite-2:19.3.0-4775.g7f24e589.el9.x86_64 2/4 2024-09-14T08:30:57.250 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : socat-1.7.4.1-6.el9.x86_64 3/4 2024-09-14T08:30:57.459 INFO:teuthology.orchestra.run.smithi164.stdout:Last metadata expiration check: 0:00:41 ago on Sat 14 Sep 2024 08:30:16 AM UTC. 2024-09-14T08:30:57.533 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 4/4 2024-09-14T08:30:57.533 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:30:57.533 INFO:teuthology.orchestra.run.smithi135.stdout:Installed: 2024-09-14T08:30:57.533 INFO:teuthology.orchestra.run.smithi135.stdout: ceph-test-2:19.3.0-4775.g7f24e589.el9.x86_64 2024-09-14T08:30:57.533 INFO:teuthology.orchestra.run.smithi135.stdout: libcephsqlite-2:19.3.0-4775.g7f24e589.el9.x86_64 2024-09-14T08:30:57.533 INFO:teuthology.orchestra.run.smithi135.stdout: socat-1.7.4.1-6.el9.x86_64 2024-09-14T08:30:57.534 INFO:teuthology.orchestra.run.smithi135.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2024-09-14T08:30:57.534 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:30:57.534 INFO:teuthology.orchestra.run.smithi135.stdout:Complete! 2024-09-14T08:30:57.613 INFO:teuthology.orchestra.run.smithi164.stdout:Dependencies resolved. 2024-09-14T08:30:57.616 INFO:teuthology.orchestra.run.smithi164.stdout:================================================================================ 2024-09-14T08:30:57.616 INFO:teuthology.orchestra.run.smithi164.stdout: Package Arch Version Repository Size 2024-09-14T08:30:57.616 INFO:teuthology.orchestra.run.smithi164.stdout:================================================================================ 2024-09-14T08:30:57.616 INFO:teuthology.orchestra.run.smithi164.stdout:Installing: 2024-09-14T08:30:57.617 INFO:teuthology.orchestra.run.smithi164.stdout: ceph x86_64 2:19.3.0-4775.g7f24e589.el9 ceph 6.6 k 2024-09-14T08:30:57.617 INFO:teuthology.orchestra.run.smithi164.stdout:Installing dependencies: 2024-09-14T08:30:57.617 INFO:teuthology.orchestra.run.smithi164.stdout: ceph-mds x86_64 2:19.3.0-4775.g7f24e589.el9 ceph 2.4 M 2024-09-14T08:30:57.617 INFO:teuthology.orchestra.run.smithi164.stdout: ceph-mgr x86_64 2:19.3.0-4775.g7f24e589.el9 ceph 1.5 M 2024-09-14T08:30:57.617 INFO:teuthology.orchestra.run.smithi164.stdout: ceph-mgr-modules-core noarch 2:19.3.0-4775.g7f24e589.el9 ceph-noarch 291 k 2024-09-14T08:30:57.617 INFO:teuthology.orchestra.run.smithi164.stdout: ceph-mon x86_64 2:19.3.0-4775.g7f24e589.el9 ceph 4.8 M 2024-09-14T08:30:57.617 INFO:teuthology.orchestra.run.smithi164.stdout: ceph-osd x86_64 2:19.3.0-4775.g7f24e589.el9 ceph 17 M 2024-09-14T08:30:57.617 INFO:teuthology.orchestra.run.smithi164.stdout: lua x86_64 5.4.4-4.el9 appstream 188 k 2024-09-14T08:30:57.617 INFO:teuthology.orchestra.run.smithi164.stdout: lua-devel x86_64 5.4.4-4.el9 crb 22 k 2024-09-14T08:30:57.617 INFO:teuthology.orchestra.run.smithi164.stdout: luarocks noarch 3.9.2-5.el9 epel 151 k 2024-09-14T08:30:57.617 INFO:teuthology.orchestra.run.smithi164.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2024-09-14T08:30:57.617 INFO:teuthology.orchestra.run.smithi164.stdout: python3-cffi x86_64 1.14.5-5.el9 baseos 253 k 2024-09-14T08:30:57.617 INFO:teuthology.orchestra.run.smithi164.stdout: python3-cheroot noarch 8.6.0-4.el9 epel 172 k 2024-09-14T08:30:57.618 INFO:teuthology.orchestra.run.smithi164.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2024-09-14T08:30:57.618 INFO:teuthology.orchestra.run.smithi164.stdout: python3-cryptography x86_64 36.0.1-4.el9 baseos 1.2 M 2024-09-14T08:30:57.618 INFO:teuthology.orchestra.run.smithi164.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2024-09-14T08:30:57.618 INFO:teuthology.orchestra.run.smithi164.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2024-09-14T08:30:57.618 INFO:teuthology.orchestra.run.smithi164.stdout: python3-jaraco-collections 2024-09-14T08:30:57.618 INFO:teuthology.orchestra.run.smithi164.stdout: noarch 3.0.0-8.el9 epel 23 k 2024-09-14T08:30:57.618 INFO:teuthology.orchestra.run.smithi164.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2024-09-14T08:30:57.618 INFO:teuthology.orchestra.run.smithi164.stdout: python3-jaraco-text noarch 3.2.0-6.el9 epel 20 k 2024-09-14T08:30:57.618 INFO:teuthology.orchestra.run.smithi164.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2024-09-14T08:30:57.618 INFO:teuthology.orchestra.run.smithi164.stdout: python3-mako noarch 1.1.4-6.el9 appstream 172 k 2024-09-14T08:30:57.618 INFO:teuthology.orchestra.run.smithi164.stdout: python3-markupsafe x86_64 1.1.1-12.el9 appstream 35 k 2024-09-14T08:30:57.618 INFO:teuthology.orchestra.run.smithi164.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2024-09-14T08:30:57.618 INFO:teuthology.orchestra.run.smithi164.stdout: python3-packaging noarch 20.9-5.el9 appstream 77 k 2024-09-14T08:30:57.619 INFO:teuthology.orchestra.run.smithi164.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2024-09-14T08:30:57.619 INFO:teuthology.orchestra.run.smithi164.stdout: python3-ply noarch 3.11-14.el9 baseos 106 k 2024-09-14T08:30:57.619 INFO:teuthology.orchestra.run.smithi164.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2024-09-14T08:30:57.619 INFO:teuthology.orchestra.run.smithi164.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2024-09-14T08:30:57.619 INFO:teuthology.orchestra.run.smithi164.stdout: python3-pycparser noarch 2.20-6.el9 baseos 135 k 2024-09-14T08:30:57.619 INFO:teuthology.orchestra.run.smithi164.stdout: python3-pytz noarch 2021.1-5.el9 appstream 51 k 2024-09-14T08:30:57.619 INFO:teuthology.orchestra.run.smithi164.stdout: python3-requests noarch 2.25.1-8.el9 baseos 125 k 2024-09-14T08:30:57.619 INFO:teuthology.orchestra.run.smithi164.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2024-09-14T08:30:57.619 INFO:teuthology.orchestra.run.smithi164.stdout: python3-toml noarch 0.10.2-6.el9 appstream 42 k 2024-09-14T08:30:57.619 INFO:teuthology.orchestra.run.smithi164.stdout: python3-urllib3 noarch 1.26.5-6.el9 baseos 215 k 2024-09-14T08:30:57.619 INFO:teuthology.orchestra.run.smithi164.stdout: python3-webob noarch 1.8.8-2.el9 epel 230 k 2024-09-14T08:30:57.619 INFO:teuthology.orchestra.run.smithi164.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2024-09-14T08:30:57.619 INFO:teuthology.orchestra.run.smithi164.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2024-09-14T08:30:57.619 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:30:57.620 INFO:teuthology.orchestra.run.smithi164.stdout:Transaction Summary 2024-09-14T08:30:57.620 INFO:teuthology.orchestra.run.smithi164.stdout:================================================================================ 2024-09-14T08:30:57.620 INFO:teuthology.orchestra.run.smithi164.stdout:Install 37 Packages 2024-09-14T08:30:57.620 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:30:57.620 INFO:teuthology.orchestra.run.smithi164.stdout:Total download size: 31 M 2024-09-14T08:30:57.620 INFO:teuthology.orchestra.run.smithi164.stdout:Installed size: 109 M 2024-09-14T08:30:57.621 INFO:teuthology.orchestra.run.smithi164.stdout:Downloading Packages: 2024-09-14T08:30:57.842 DEBUG:teuthology.orchestra.run.smithi135:> sudo yum -y install ceph 2024-09-14T08:30:58.075 INFO:teuthology.orchestra.run.smithi164.stdout:(1/37): ceph-19.3.0-4775.g7f24e589.el9.x86_64.r 52 kB/s | 6.6 kB 00:00 2024-09-14T08:30:58.292 INFO:teuthology.orchestra.run.smithi164.stdout:(2/37): ceph-mds-19.3.0-4775.g7f24e589.el9.x86_ 6.9 MB/s | 2.4 MB 00:00 2024-09-14T08:30:58.334 INFO:teuthology.orchestra.run.smithi164.stdout:(3/37): ceph-mgr-19.3.0-4775.g7f24e589.el9.x86_ 4.0 MB/s | 1.5 MB 00:00 2024-09-14T08:30:58.401 INFO:teuthology.orchestra.run.smithi164.stdout:(4/37): ceph-mgr-modules-core-19.3.0-4775.g7f24 4.2 MB/s | 291 kB 00:00 2024-09-14T08:30:58.448 INFO:teuthology.orchestra.run.smithi135.stdout:Last metadata expiration check: 0:00:41 ago on Sat 14 Sep 2024 08:30:17 AM UTC. 2024-09-14T08:30:58.484 INFO:teuthology.orchestra.run.smithi164.stdout:(5/37): ceph-mon-19.3.0-4775.g7f24e589.el9.x86_ 12 MB/s | 4.8 MB 00:00 2024-09-14T08:30:58.600 INFO:teuthology.orchestra.run.smithi135.stdout:Dependencies resolved. 2024-09-14T08:30:58.603 INFO:teuthology.orchestra.run.smithi135.stdout:================================================================================ 2024-09-14T08:30:58.603 INFO:teuthology.orchestra.run.smithi135.stdout: Package Arch Version Repository Size 2024-09-14T08:30:58.603 INFO:teuthology.orchestra.run.smithi135.stdout:================================================================================ 2024-09-14T08:30:58.603 INFO:teuthology.orchestra.run.smithi135.stdout:Installing: 2024-09-14T08:30:58.603 INFO:teuthology.orchestra.run.smithi135.stdout: ceph x86_64 2:19.3.0-4775.g7f24e589.el9 ceph 6.6 k 2024-09-14T08:30:58.603 INFO:teuthology.orchestra.run.smithi135.stdout:Installing dependencies: 2024-09-14T08:30:58.603 INFO:teuthology.orchestra.run.smithi135.stdout: ceph-mds x86_64 2:19.3.0-4775.g7f24e589.el9 ceph 2.4 M 2024-09-14T08:30:58.603 INFO:teuthology.orchestra.run.smithi135.stdout: ceph-mgr x86_64 2:19.3.0-4775.g7f24e589.el9 ceph 1.5 M 2024-09-14T08:30:58.603 INFO:teuthology.orchestra.run.smithi135.stdout: ceph-mgr-modules-core noarch 2:19.3.0-4775.g7f24e589.el9 ceph-noarch 291 k 2024-09-14T08:30:58.603 INFO:teuthology.orchestra.run.smithi135.stdout: ceph-mon x86_64 2:19.3.0-4775.g7f24e589.el9 ceph 4.8 M 2024-09-14T08:30:58.604 INFO:teuthology.orchestra.run.smithi135.stdout: ceph-osd x86_64 2:19.3.0-4775.g7f24e589.el9 ceph 17 M 2024-09-14T08:30:58.604 INFO:teuthology.orchestra.run.smithi135.stdout: lua x86_64 5.4.4-4.el9 appstream 188 k 2024-09-14T08:30:58.604 INFO:teuthology.orchestra.run.smithi135.stdout: lua-devel x86_64 5.4.4-4.el9 crb 22 k 2024-09-14T08:30:58.604 INFO:teuthology.orchestra.run.smithi135.stdout: luarocks noarch 3.9.2-5.el9 epel 151 k 2024-09-14T08:30:58.604 INFO:teuthology.orchestra.run.smithi135.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2024-09-14T08:30:58.604 INFO:teuthology.orchestra.run.smithi135.stdout: python3-cffi x86_64 1.14.5-5.el9 baseos 253 k 2024-09-14T08:30:58.604 INFO:teuthology.orchestra.run.smithi135.stdout: python3-cheroot noarch 8.6.0-4.el9 epel 172 k 2024-09-14T08:30:58.604 INFO:teuthology.orchestra.run.smithi135.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2024-09-14T08:30:58.604 INFO:teuthology.orchestra.run.smithi135.stdout: python3-cryptography x86_64 36.0.1-4.el9 baseos 1.2 M 2024-09-14T08:30:58.604 INFO:teuthology.orchestra.run.smithi135.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2024-09-14T08:30:58.604 INFO:teuthology.orchestra.run.smithi135.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2024-09-14T08:30:58.604 INFO:teuthology.orchestra.run.smithi135.stdout: python3-jaraco-collections 2024-09-14T08:30:58.604 INFO:teuthology.orchestra.run.smithi135.stdout: noarch 3.0.0-8.el9 epel 23 k 2024-09-14T08:30:58.605 INFO:teuthology.orchestra.run.smithi135.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2024-09-14T08:30:58.605 INFO:teuthology.orchestra.run.smithi135.stdout: python3-jaraco-text noarch 3.2.0-6.el9 epel 20 k 2024-09-14T08:30:58.605 INFO:teuthology.orchestra.run.smithi135.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2024-09-14T08:30:58.605 INFO:teuthology.orchestra.run.smithi135.stdout: python3-mako noarch 1.1.4-6.el9 appstream 172 k 2024-09-14T08:30:58.605 INFO:teuthology.orchestra.run.smithi135.stdout: python3-markupsafe x86_64 1.1.1-12.el9 appstream 35 k 2024-09-14T08:30:58.605 INFO:teuthology.orchestra.run.smithi135.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2024-09-14T08:30:58.605 INFO:teuthology.orchestra.run.smithi135.stdout: python3-packaging noarch 20.9-5.el9 appstream 77 k 2024-09-14T08:30:58.605 INFO:teuthology.orchestra.run.smithi135.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2024-09-14T08:30:58.605 INFO:teuthology.orchestra.run.smithi135.stdout: python3-ply noarch 3.11-14.el9 baseos 106 k 2024-09-14T08:30:58.605 INFO:teuthology.orchestra.run.smithi135.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2024-09-14T08:30:58.605 INFO:teuthology.orchestra.run.smithi135.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2024-09-14T08:30:58.605 INFO:teuthology.orchestra.run.smithi135.stdout: python3-pycparser noarch 2.20-6.el9 baseos 135 k 2024-09-14T08:30:58.605 INFO:teuthology.orchestra.run.smithi135.stdout: python3-pytz noarch 2021.1-5.el9 appstream 51 k 2024-09-14T08:30:58.605 INFO:teuthology.orchestra.run.smithi135.stdout: python3-requests noarch 2.25.1-8.el9 baseos 125 k 2024-09-14T08:30:58.606 INFO:teuthology.orchestra.run.smithi135.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2024-09-14T08:30:58.606 INFO:teuthology.orchestra.run.smithi135.stdout: python3-toml noarch 0.10.2-6.el9 appstream 42 k 2024-09-14T08:30:58.606 INFO:teuthology.orchestra.run.smithi135.stdout: python3-urllib3 noarch 1.26.5-6.el9 baseos 215 k 2024-09-14T08:30:58.606 INFO:teuthology.orchestra.run.smithi135.stdout: python3-webob noarch 1.8.8-2.el9 epel 230 k 2024-09-14T08:30:58.606 INFO:teuthology.orchestra.run.smithi135.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2024-09-14T08:30:58.606 INFO:teuthology.orchestra.run.smithi135.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2024-09-14T08:30:58.606 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:30:58.606 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction Summary 2024-09-14T08:30:58.606 INFO:teuthology.orchestra.run.smithi135.stdout:================================================================================ 2024-09-14T08:30:58.606 INFO:teuthology.orchestra.run.smithi135.stdout:Install 37 Packages 2024-09-14T08:30:58.606 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:30:58.607 INFO:teuthology.orchestra.run.smithi135.stdout:Total download size: 31 M 2024-09-14T08:30:58.607 INFO:teuthology.orchestra.run.smithi135.stdout:Installed size: 109 M 2024-09-14T08:30:58.607 INFO:teuthology.orchestra.run.smithi135.stdout:Downloading Packages: 2024-09-14T08:30:58.652 INFO:teuthology.orchestra.run.smithi164.stdout:(6/37): python3-cffi-1.14.5-5.el9.x86_64.rpm 1.0 MB/s | 253 kB 00:00 2024-09-14T08:30:58.701 INFO:teuthology.orchestra.run.smithi164.stdout:(7/37): python3-ply-3.11-14.el9.noarch.rpm 2.1 MB/s | 106 kB 00:00 2024-09-14T08:30:58.760 INFO:teuthology.orchestra.run.smithi164.stdout:(8/37): python3-pycparser-2.20-6.el9.noarch.rpm 2.3 MB/s | 135 kB 00:00 2024-09-14T08:30:58.977 INFO:teuthology.orchestra.run.smithi164.stdout:(9/37): ceph-osd-19.3.0-4775.g7f24e589.el9.x86_ 25 MB/s | 17 MB 00:00 2024-09-14T08:30:59.002 INFO:teuthology.orchestra.run.smithi164.stdout:(10/37): python3-requests-2.25.1-8.el9.noarch.r 516 kB/s | 125 kB 00:00 2024-09-14T08:30:59.045 INFO:teuthology.orchestra.run.smithi164.stdout:(11/37): python3-cryptography-36.0.1-4.el9.x86_ 2.2 MB/s | 1.2 MB 00:00 2024-09-14T08:30:59.211 INFO:teuthology.orchestra.run.smithi164.stdout:(12/37): python3-urllib3-1.26.5-6.el9.noarch.rp 919 kB/s | 215 kB 00:00 2024-09-14T08:30:59.260 INFO:teuthology.orchestra.run.smithi135.stdout:(1/37): ceph-19.3.0-4775.g7f24e589.el9.x86_64.r 44 kB/s | 6.6 kB 00:00 2024-09-14T08:30:59.444 INFO:teuthology.orchestra.run.smithi135.stdout:(2/37): ceph-mgr-19.3.0-4775.g7f24e589.el9.x86_ 4.6 MB/s | 1.5 MB 00:00 2024-09-14T08:30:59.494 INFO:teuthology.orchestra.run.smithi135.stdout:(3/37): ceph-mds-19.3.0-4775.g7f24e589.el9.x86_ 6.2 MB/s | 2.4 MB 00:00 2024-09-14T08:30:59.553 INFO:teuthology.orchestra.run.smithi135.stdout:(4/37): ceph-mgr-modules-core-19.3.0-4775.g7f24 4.9 MB/s | 291 kB 00:00 2024-09-14T08:30:59.703 INFO:teuthology.orchestra.run.smithi135.stdout:(5/37): ceph-mon-19.3.0-4775.g7f24e589.el9.x86_ 11 MB/s | 4.8 MB 00:00 2024-09-14T08:30:59.770 INFO:teuthology.orchestra.run.smithi164.stdout:(13/37): python3-markupsafe-1.1.1-12.el9.x86_64 62 kB/s | 35 kB 00:00 2024-09-14T08:30:59.795 INFO:teuthology.orchestra.run.smithi164.stdout:(14/37): lua-5.4.4-4.el9.x86_64.rpm 238 kB/s | 188 kB 00:00 2024-09-14T08:30:59.829 INFO:teuthology.orchestra.run.smithi164.stdout:(15/37): python3-mako-1.1.4-6.el9.noarch.rpm 219 kB/s | 172 kB 00:00 2024-09-14T08:31:00.070 INFO:teuthology.orchestra.run.smithi135.stdout:(6/37): ceph-osd-19.3.0-4775.g7f24e589.el9.x86_ 27 MB/s | 17 MB 00:00 2024-09-14T08:31:00.396 INFO:teuthology.orchestra.run.smithi164.stdout:(16/37): python3-pytz-2021.1-5.el9.noarch.rpm 85 kB/s | 51 kB 00:00 2024-09-14T08:31:00.421 INFO:teuthology.orchestra.run.smithi164.stdout:(17/37): python3-toml-0.10.2-6.el9.noarch.rpm 70 kB/s | 42 kB 00:00 2024-09-14T08:31:00.447 INFO:teuthology.orchestra.run.smithi164.stdout:(18/37): python3-packaging-20.9-5.el9.noarch.rp 114 kB/s | 77 kB 00:00 2024-09-14T08:31:00.470 INFO:teuthology.orchestra.run.smithi135.stdout:(7/37): python3-cffi-1.14.5-5.el9.x86_64.rpm 276 kB/s | 253 kB 00:00 2024-09-14T08:31:00.489 INFO:teuthology.orchestra.run.smithi164.stdout:(19/37): luarocks-3.9.2-5.el9.noarch.rpm 2.3 MB/s | 151 kB 00:00 2024-09-14T08:31:00.514 INFO:teuthology.orchestra.run.smithi164.stdout:(20/37): python3-bcrypt-3.2.2-1.el9.x86_64.rpm 650 kB/s | 43 kB 00:00 2024-09-14T08:31:00.539 INFO:teuthology.orchestra.run.smithi164.stdout:(21/37): python3-cheroot-8.6.0-4.el9.noarch.rpm 3.5 MB/s | 172 kB 00:00 2024-09-14T08:31:00.564 INFO:teuthology.orchestra.run.smithi164.stdout:(22/37): lua-devel-5.4.4-4.el9.x86_64.rpm 158 kB/s | 22 kB 00:00 2024-09-14T08:31:00.590 INFO:teuthology.orchestra.run.smithi164.stdout:(23/37): python3-jaraco-8.2.1-3.el9.noarch.rpm 213 kB/s | 11 kB 00:00 2024-09-14T08:31:00.624 INFO:teuthology.orchestra.run.smithi164.stdout:(24/37): python3-jaraco-collections-3.0.0-8.el9 894 kB/s | 23 kB 00:00 2024-09-14T08:31:00.648 INFO:teuthology.orchestra.run.smithi164.stdout:(25/37): python3-jaraco-functools-3.5.0-2.el9.n 849 kB/s | 19 kB 00:00 2024-09-14T08:31:00.674 INFO:teuthology.orchestra.run.smithi164.stdout:(26/37): python3-jaraco-classes-3.2.1-5.el9.noa 162 kB/s | 18 kB 00:00 2024-09-14T08:31:00.699 INFO:teuthology.orchestra.run.smithi164.stdout:(27/37): python3-jaraco-text-3.2.0-6.el9.noarch 390 kB/s | 20 kB 00:00 2024-09-14T08:31:00.725 INFO:teuthology.orchestra.run.smithi164.stdout:(28/37): python3-logutils-0.3.5-21.el9.noarch.r 921 kB/s | 46 kB 00:00 2024-09-14T08:31:00.746 INFO:teuthology.orchestra.run.smithi135.stdout:(8/37): python3-ply-3.11-14.el9.noarch.rpm 159 kB/s | 106 kB 00:00 2024-09-14T08:31:00.750 INFO:teuthology.orchestra.run.smithi164.stdout:(29/37): python3-cherrypy-18.6.1-2.el9.noarch.r 1.5 MB/s | 358 kB 00:00 2024-09-14T08:31:00.775 INFO:teuthology.orchestra.run.smithi164.stdout:(30/37): python3-more-itertools-8.12.0-2.el9.no 1.0 MB/s | 79 kB 00:00 2024-09-14T08:31:00.801 INFO:teuthology.orchestra.run.smithi164.stdout:(31/37): python3-portend-3.1.0-2.el9.noarch.rpm 328 kB/s | 16 kB 00:00 2024-09-14T08:31:00.826 INFO:teuthology.orchestra.run.smithi164.stdout:(32/37): python3-tempora-5.0.0-2.el9.noarch.rpm 1.4 MB/s | 36 kB 00:00 2024-09-14T08:31:00.851 INFO:teuthology.orchestra.run.smithi164.stdout:(33/37): python3-pyOpenSSL-21.0.0-1.el9.noarch. 1.8 MB/s | 90 kB 00:00 2024-09-14T08:31:00.876 INFO:teuthology.orchestra.run.smithi164.stdout:(34/37): python3-pecan-1.4.2-3.el9.noarch.rpm 1.8 MB/s | 272 kB 00:00 2024-09-14T08:31:00.901 INFO:teuthology.orchestra.run.smithi164.stdout:(35/37): python3-zc-lockfile-2.0-10.el9.noarch. 808 kB/s | 20 kB 00:00 2024-09-14T08:31:00.904 INFO:teuthology.orchestra.run.smithi135.stdout:(9/37): python3-cryptography-36.0.1-4.el9.x86_6 1.0 MB/s | 1.2 MB 00:01 2024-09-14T08:31:00.927 INFO:teuthology.orchestra.run.smithi164.stdout:(36/37): python3-webob-1.8.8-2.el9.noarch.rpm 2.2 MB/s | 230 kB 00:00 2024-09-14T08:31:00.952 INFO:teuthology.orchestra.run.smithi164.stdout:(37/37): python3-werkzeug-2.0.3-3.el9.1.noarch. 4.2 MB/s | 427 kB 00:00 2024-09-14T08:31:00.957 INFO:teuthology.orchestra.run.smithi164.stdout:-------------------------------------------------------------------------------- 2024-09-14T08:31:00.957 INFO:teuthology.orchestra.run.smithi164.stdout:Total 9.2 MB/s | 31 MB 00:03 2024-09-14T08:31:01.108 INFO:teuthology.orchestra.run.smithi164.stdout:Running transaction check 2024-09-14T08:31:01.149 INFO:teuthology.orchestra.run.smithi164.stdout:Transaction check succeeded. 2024-09-14T08:31:01.149 INFO:teuthology.orchestra.run.smithi164.stdout:Running transaction test 2024-09-14T08:31:01.163 INFO:teuthology.orchestra.run.smithi135.stdout:(10/37): python3-pycparser-2.20-6.el9.noarch.rp 195 kB/s | 135 kB 00:00 2024-09-14T08:31:01.371 INFO:teuthology.orchestra.run.smithi135.stdout:(11/37): lua-5.4.4-4.el9.x86_64.rpm 906 kB/s | 188 kB 00:00 2024-09-14T08:31:01.455 INFO:teuthology.orchestra.run.smithi135.stdout:(12/37): python3-requests-2.25.1-8.el9.noarch.r 176 kB/s | 125 kB 00:00 2024-09-14T08:31:01.480 INFO:teuthology.orchestra.run.smithi135.stdout:(13/37): python3-mako-1.1.4-6.el9.noarch.rpm 1.5 MB/s | 172 kB 00:00 2024-09-14T08:31:01.530 INFO:teuthology.orchestra.run.smithi135.stdout:(14/37): python3-packaging-20.9-5.el9.noarch.rp 1.5 MB/s | 77 kB 00:00 2024-09-14T08:31:01.564 INFO:teuthology.orchestra.run.smithi135.stdout:(15/37): python3-markupsafe-1.1.1-12.el9.x86_64 319 kB/s | 35 kB 00:00 2024-09-14T08:31:01.589 INFO:teuthology.orchestra.run.smithi135.stdout:(16/37): python3-pytz-2021.1-5.el9.noarch.rpm 870 kB/s | 51 kB 00:00 2024-09-14T08:31:01.632 INFO:teuthology.orchestra.run.smithi164.stdout:Transaction test succeeded. 2024-09-14T08:31:01.632 INFO:teuthology.orchestra.run.smithi164.stdout:Running transaction 2024-09-14T08:31:01.648 INFO:teuthology.orchestra.run.smithi135.stdout:(17/37): python3-toml-0.10.2-6.el9.noarch.rpm 499 kB/s | 42 kB 00:00 2024-09-14T08:31:01.673 INFO:teuthology.orchestra.run.smithi135.stdout:(18/37): python3-urllib3-1.26.5-6.el9.noarch.rp 279 kB/s | 215 kB 00:00 2024-09-14T08:31:01.723 INFO:teuthology.orchestra.run.smithi135.stdout:(19/37): luarocks-3.9.2-5.el9.noarch.rpm 2.0 MB/s | 151 kB 00:00 2024-09-14T08:31:01.749 INFO:teuthology.orchestra.run.smithi135.stdout:(20/37): python3-bcrypt-3.2.2-1.el9.x86_64.rpm 648 kB/s | 43 kB 00:00 2024-09-14T08:31:01.776 INFO:teuthology.orchestra.run.smithi135.stdout:(21/37): python3-cheroot-8.6.0-4.el9.noarch.rpm 3.3 MB/s | 172 kB 00:00 2024-09-14T08:31:01.799 INFO:teuthology.orchestra.run.smithi135.stdout:(22/37): python3-jaraco-8.2.1-3.el9.noarch.rpm 428 kB/s | 11 kB 00:00 2024-09-14T08:31:01.824 INFO:teuthology.orchestra.run.smithi135.stdout:(23/37): python3-jaraco-classes-3.2.1-5.el9.noa 711 kB/s | 18 kB 00:00 2024-09-14T08:31:01.858 INFO:teuthology.orchestra.run.smithi135.stdout:(24/37): python3-cherrypy-18.6.1-2.el9.noarch.r 3.2 MB/s | 358 kB 00:00 2024-09-14T08:31:01.883 INFO:teuthology.orchestra.run.smithi135.stdout:(25/37): python3-jaraco-collections-3.0.0-8.el9 452 kB/s | 23 kB 00:00 2024-09-14T08:31:01.908 INFO:teuthology.orchestra.run.smithi135.stdout:(26/37): python3-jaraco-functools-3.5.0-2.el9.n 389 kB/s | 19 kB 00:00 2024-09-14T08:31:01.925 INFO:teuthology.orchestra.run.smithi135.stdout:(27/37): python3-jaraco-text-3.2.0-6.el9.noarch 471 kB/s | 20 kB 00:00 2024-09-14T08:31:01.951 INFO:teuthology.orchestra.run.smithi135.stdout:(28/37): python3-logutils-0.3.5-21.el9.noarch.r 1.1 MB/s | 46 kB 00:00 2024-09-14T08:31:01.976 INFO:teuthology.orchestra.run.smithi135.stdout:(29/37): python3-more-itertools-8.12.0-2.el9.no 1.5 MB/s | 79 kB 00:00 2024-09-14T08:31:01.993 INFO:teuthology.orchestra.run.smithi135.stdout:(30/37): python3-portend-3.1.0-2.el9.noarch.rpm 995 kB/s | 16 kB 00:00 2024-09-14T08:31:02.027 INFO:teuthology.orchestra.run.smithi135.stdout:(31/37): python3-pecan-1.4.2-3.el9.noarch.rpm 3.5 MB/s | 272 kB 00:00 2024-09-14T08:31:02.052 INFO:teuthology.orchestra.run.smithi135.stdout:(32/37): python3-pyOpenSSL-21.0.0-1.el9.noarch. 1.5 MB/s | 90 kB 00:00 2024-09-14T08:31:02.078 INFO:teuthology.orchestra.run.smithi135.stdout:(33/37): python3-tempora-5.0.0-2.el9.noarch.rpm 710 kB/s | 36 kB 00:00 2024-09-14T08:31:02.103 INFO:teuthology.orchestra.run.smithi135.stdout:(34/37): python3-webob-1.8.8-2.el9.noarch.rpm 4.5 MB/s | 230 kB 00:00 2024-09-14T08:31:02.128 INFO:teuthology.orchestra.run.smithi135.stdout:(35/37): python3-zc-lockfile-2.0-10.el9.noarch. 800 kB/s | 20 kB 00:00 2024-09-14T08:31:02.161 INFO:teuthology.orchestra.run.smithi135.stdout:(36/37): python3-werkzeug-2.0.3-3.el9.1.noarch. 5.0 MB/s | 427 kB 00:00 2024-09-14T08:31:02.186 INFO:teuthology.orchestra.run.smithi135.stdout:(37/37): lua-devel-5.4.4-4.el9.x86_64.rpm 37 kB/s | 22 kB 00:00 2024-09-14T08:31:02.193 INFO:teuthology.orchestra.run.smithi135.stdout:-------------------------------------------------------------------------------- 2024-09-14T08:31:02.193 INFO:teuthology.orchestra.run.smithi135.stdout:Total 8.6 MB/s | 31 MB 00:03 2024-09-14T08:31:02.343 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction check 2024-09-14T08:31:02.384 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction check succeeded. 2024-09-14T08:31:02.385 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction test 2024-09-14T08:31:02.857 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction test succeeded. 2024-09-14T08:31:02.857 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction 2024-09-14T08:31:03.029 INFO:teuthology.orchestra.run.smithi164.stdout: Preparing : 1/1 2024-09-14T08:31:03.088 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 1/37 2024-09-14T08:31:03.121 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 2/37 2024-09-14T08:31:03.166 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : lua-5.4.4-4.el9.x86_64 3/37 2024-09-14T08:31:03.237 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : lua-devel-5.4.4-4.el9.x86_64 4/37 2024-09-14T08:31:03.282 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : luarocks-3.9.2-5.el9.noarch 5/37 2024-09-14T08:31:03.314 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 6/37 2024-09-14T08:31:03.406 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 7/37 2024-09-14T08:31:03.485 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 8/37 2024-09-14T08:31:03.526 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : python3-webob-1.8.8-2.el9.noarch 9/37 2024-09-14T08:31:03.561 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 10/37 2024-09-14T08:31:03.593 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 11/37 2024-09-14T08:31:03.627 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 12/37 2024-09-14T08:31:03.661 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : python3-jaraco-text-3.2.0-6.el9.noarch 13/37 2024-09-14T08:31:03.702 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 14/37 2024-09-14T08:31:03.745 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : python3-pytz-2021.1-5.el9.noarch 15/37 2024-09-14T08:31:03.788 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 16/37 2024-09-14T08:31:03.836 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 17/37 2024-09-14T08:31:03.868 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : python3-packaging-20.9-5.el9.noarch 18/37 2024-09-14T08:31:03.941 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 19/37 2024-09-14T08:31:04.082 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 20/37 2024-09-14T08:31:04.168 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 21/37 2024-09-14T08:31:04.221 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : python3-urllib3-1.26.5-6.el9.noarch 22/37 2024-09-14T08:31:04.260 INFO:teuthology.orchestra.run.smithi135.stdout: Preparing : 1/1 2024-09-14T08:31:04.268 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : python3-requests-2.25.1-8.el9.noarch 23/37 2024-09-14T08:31:04.325 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : python3-ply-3.11-14.el9.noarch 24/37 2024-09-14T08:31:04.331 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 1/37 2024-09-14T08:31:04.364 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 2/37 2024-09-14T08:31:04.381 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 25/37 2024-09-14T08:31:04.400 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : lua-5.4.4-4.el9.x86_64 3/37 2024-09-14T08:31:04.464 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : lua-devel-5.4.4-4.el9.x86_64 4/37 2024-09-14T08:31:04.508 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : luarocks-3.9.2-5.el9.noarch 5/37 2024-09-14T08:31:04.531 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 6/37 2024-09-14T08:31:04.567 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 26/37 2024-09-14T08:31:04.615 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 7/37 2024-09-14T08:31:04.622 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : python3-cryptography-36.0.1-4.el9.x86_64 27/37 2024-09-14T08:31:04.703 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 8/37 2024-09-14T08:31:04.712 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 28/37 2024-09-14T08:31:04.752 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-webob-1.8.8-2.el9.noarch 9/37 2024-09-14T08:31:04.787 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 10/37 2024-09-14T08:31:04.805 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : python3-cheroot-8.6.0-4.el9.noarch 29/37 2024-09-14T08:31:04.816 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 11/37 2024-09-14T08:31:04.850 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 12/37 2024-09-14T08:31:04.863 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 30/37 2024-09-14T08:31:04.893 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-jaraco-text-3.2.0-6.el9.noarch 13/37 2024-09-14T08:31:04.934 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 14/37 2024-09-14T08:31:04.977 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-pytz-2021.1-5.el9.noarch 15/37 2024-09-14T08:31:04.991 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 31/37 2024-09-14T08:31:05.011 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 16/37 2024-09-14T08:31:05.060 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 17/37 2024-09-14T08:31:05.092 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-packaging-20.9-5.el9.noarch 18/37 2024-09-14T08:31:05.160 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : ceph-mgr-modules-core-2:19.3.0-4775.g7f24e589.el9. 32/37 2024-09-14T08:31:05.165 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 19/37 2024-09-14T08:31:05.190 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : ceph-mgr-2:19.3.0-4775.g7f24e589.el9.x86_64 33/37 2024-09-14T08:31:05.216 INFO:teuthology.orchestra.run.smithi164.stdout: Running scriptlet: ceph-mgr-2:19.3.0-4775.g7f24e589.el9.x86_64 33/37 2024-09-14T08:31:05.216 INFO:teuthology.orchestra.run.smithi164.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-14T08:31:05.216 INFO:teuthology.orchestra.run.smithi164.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2024-09-14T08:31:05.216 INFO:teuthology.orchestra.run.smithi164.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-09-14T08:31:05.216 INFO:teuthology.orchestra.run.smithi164.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-09-14T08:31:05.216 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:31:05.304 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 20/37 2024-09-14T08:31:05.391 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 21/37 2024-09-14T08:31:05.444 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-urllib3-1.26.5-6.el9.noarch 22/37 2024-09-14T08:31:05.483 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-requests-2.25.1-8.el9.noarch 23/37 2024-09-14T08:31:05.540 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-ply-3.11-14.el9.noarch 24/37 2024-09-14T08:31:05.596 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 25/37 2024-09-14T08:31:05.789 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 26/37 2024-09-14T08:31:05.845 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-cryptography-36.0.1-4.el9.x86_64 27/37 2024-09-14T08:31:05.924 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 28/37 2024-09-14T08:31:06.012 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-cheroot-8.6.0-4.el9.noarch 29/37 2024-09-14T08:31:06.053 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 30/37 2024-09-14T08:31:06.183 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 31/37 2024-09-14T08:31:06.353 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : ceph-mgr-modules-core-2:19.3.0-4775.g7f24e589.el9. 32/37 2024-09-14T08:31:06.379 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : ceph-mgr-2:19.3.0-4775.g7f24e589.el9.x86_64 33/37 2024-09-14T08:31:06.406 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: ceph-mgr-2:19.3.0-4775.g7f24e589.el9.x86_64 33/37 2024-09-14T08:31:06.407 INFO:teuthology.orchestra.run.smithi135.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-14T08:31:06.407 INFO:teuthology.orchestra.run.smithi135.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2024-09-14T08:31:06.407 INFO:teuthology.orchestra.run.smithi135.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-09-14T08:31:06.407 INFO:teuthology.orchestra.run.smithi135.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-09-14T08:31:06.407 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:31:06.683 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : ceph-osd-2:19.3.0-4775.g7f24e589.el9.x86_64 34/37 2024-09-14T08:31:06.710 INFO:teuthology.orchestra.run.smithi164.stdout: Running scriptlet: ceph-osd-2:19.3.0-4775.g7f24e589.el9.x86_64 34/37 2024-09-14T08:31:06.711 INFO:teuthology.orchestra.run.smithi164.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-14T08:31:06.711 INFO:teuthology.orchestra.run.smithi164.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2024-09-14T08:31:06.711 INFO:teuthology.orchestra.run.smithi164.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-09-14T08:31:06.711 INFO:teuthology.orchestra.run.smithi164.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-09-14T08:31:06.711 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:31:07.167 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : ceph-mon-2:19.3.0-4775.g7f24e589.el9.x86_64 35/37 2024-09-14T08:31:07.190 INFO:teuthology.orchestra.run.smithi164.stdout: Running scriptlet: ceph-mon-2:19.3.0-4775.g7f24e589.el9.x86_64 35/37 2024-09-14T08:31:07.191 INFO:teuthology.orchestra.run.smithi164.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-14T08:31:07.191 INFO:teuthology.orchestra.run.smithi164.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2024-09-14T08:31:07.191 INFO:teuthology.orchestra.run.smithi164.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-09-14T08:31:07.191 INFO:teuthology.orchestra.run.smithi164.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-09-14T08:31:07.191 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:31:07.418 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : ceph-mds-2:19.3.0-4775.g7f24e589.el9.x86_64 36/37 2024-09-14T08:31:07.441 INFO:teuthology.orchestra.run.smithi164.stdout: Running scriptlet: ceph-mds-2:19.3.0-4775.g7f24e589.el9.x86_64 36/37 2024-09-14T08:31:07.441 INFO:teuthology.orchestra.run.smithi164.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-14T08:31:07.442 INFO:teuthology.orchestra.run.smithi164.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2024-09-14T08:31:07.442 INFO:teuthology.orchestra.run.smithi164.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-09-14T08:31:07.442 INFO:teuthology.orchestra.run.smithi164.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-09-14T08:31:07.442 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:31:07.506 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : ceph-2:19.3.0-4775.g7f24e589.el9.x86_64 37/37 2024-09-14T08:31:07.880 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : ceph-osd-2:19.3.0-4775.g7f24e589.el9.x86_64 34/37 2024-09-14T08:31:07.908 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: ceph-osd-2:19.3.0-4775.g7f24e589.el9.x86_64 34/37 2024-09-14T08:31:07.908 INFO:teuthology.orchestra.run.smithi135.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-14T08:31:07.908 INFO:teuthology.orchestra.run.smithi135.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2024-09-14T08:31:07.908 INFO:teuthology.orchestra.run.smithi135.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-09-14T08:31:07.908 INFO:teuthology.orchestra.run.smithi135.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-09-14T08:31:07.908 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:31:08.356 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : ceph-mon-2:19.3.0-4775.g7f24e589.el9.x86_64 35/37 2024-09-14T08:31:08.380 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: ceph-mon-2:19.3.0-4775.g7f24e589.el9.x86_64 35/37 2024-09-14T08:31:08.380 INFO:teuthology.orchestra.run.smithi135.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-14T08:31:08.380 INFO:teuthology.orchestra.run.smithi135.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2024-09-14T08:31:08.380 INFO:teuthology.orchestra.run.smithi135.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-09-14T08:31:08.380 INFO:teuthology.orchestra.run.smithi135.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-09-14T08:31:08.380 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:31:08.615 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : ceph-mds-2:19.3.0-4775.g7f24e589.el9.x86_64 36/37 2024-09-14T08:31:08.639 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: ceph-mds-2:19.3.0-4775.g7f24e589.el9.x86_64 36/37 2024-09-14T08:31:08.639 INFO:teuthology.orchestra.run.smithi135.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-14T08:31:08.639 INFO:teuthology.orchestra.run.smithi135.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2024-09-14T08:31:08.639 INFO:teuthology.orchestra.run.smithi135.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-09-14T08:31:08.639 INFO:teuthology.orchestra.run.smithi135.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-09-14T08:31:08.639 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:31:08.704 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : ceph-2:19.3.0-4775.g7f24e589.el9.x86_64 37/37 2024-09-14T08:31:09.367 INFO:teuthology.orchestra.run.smithi164.stdout: Running scriptlet: ceph-2:19.3.0-4775.g7f24e589.el9.x86_64 37/37 2024-09-14T08:31:09.367 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : ceph-2:19.3.0-4775.g7f24e589.el9.x86_64 1/37 2024-09-14T08:31:09.367 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : ceph-mds-2:19.3.0-4775.g7f24e589.el9.x86_64 2/37 2024-09-14T08:31:09.367 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : ceph-mgr-2:19.3.0-4775.g7f24e589.el9.x86_64 3/37 2024-09-14T08:31:09.367 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : ceph-mon-2:19.3.0-4775.g7f24e589.el9.x86_64 4/37 2024-09-14T08:31:09.367 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : ceph-osd-2:19.3.0-4775.g7f24e589.el9.x86_64 5/37 2024-09-14T08:31:09.367 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : ceph-mgr-modules-core-2:19.3.0-4775.g7f24e589.el9. 6/37 2024-09-14T08:31:09.367 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 7/37 2024-09-14T08:31:09.367 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : python3-cryptography-36.0.1-4.el9.x86_64 8/37 2024-09-14T08:31:09.367 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : python3-ply-3.11-14.el9.noarch 9/37 2024-09-14T08:31:09.368 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 10/37 2024-09-14T08:31:09.368 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : python3-requests-2.25.1-8.el9.noarch 11/37 2024-09-14T08:31:09.368 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : python3-urllib3-1.26.5-6.el9.noarch 12/37 2024-09-14T08:31:09.368 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : lua-5.4.4-4.el9.x86_64 13/37 2024-09-14T08:31:09.368 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 14/37 2024-09-14T08:31:09.368 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 15/37 2024-09-14T08:31:09.368 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : python3-packaging-20.9-5.el9.noarch 16/37 2024-09-14T08:31:09.368 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : python3-pytz-2021.1-5.el9.noarch 17/37 2024-09-14T08:31:09.368 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 18/37 2024-09-14T08:31:09.368 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : lua-devel-5.4.4-4.el9.x86_64 19/37 2024-09-14T08:31:09.368 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : luarocks-3.9.2-5.el9.noarch 20/37 2024-09-14T08:31:09.368 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 21/37 2024-09-14T08:31:09.368 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : python3-cheroot-8.6.0-4.el9.noarch 22/37 2024-09-14T08:31:09.369 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 23/37 2024-09-14T08:31:09.369 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 24/37 2024-09-14T08:31:09.369 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 25/37 2024-09-14T08:31:09.369 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 26/37 2024-09-14T08:31:09.369 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 27/37 2024-09-14T08:31:09.369 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : python3-jaraco-text-3.2.0-6.el9.noarch 28/37 2024-09-14T08:31:09.369 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 29/37 2024-09-14T08:31:09.369 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 30/37 2024-09-14T08:31:09.369 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 31/37 2024-09-14T08:31:09.369 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 32/37 2024-09-14T08:31:09.369 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 33/37 2024-09-14T08:31:09.369 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 34/37 2024-09-14T08:31:09.369 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : python3-webob-1.8.8-2.el9.noarch 35/37 2024-09-14T08:31:09.369 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 36/37 2024-09-14T08:31:10.472 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 37/37 2024-09-14T08:31:10.472 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:31:10.472 INFO:teuthology.orchestra.run.smithi164.stdout:Installed: 2024-09-14T08:31:10.472 INFO:teuthology.orchestra.run.smithi164.stdout: ceph-2:19.3.0-4775.g7f24e589.el9.x86_64 2024-09-14T08:31:10.472 INFO:teuthology.orchestra.run.smithi164.stdout: ceph-mds-2:19.3.0-4775.g7f24e589.el9.x86_64 2024-09-14T08:31:10.472 INFO:teuthology.orchestra.run.smithi164.stdout: ceph-mgr-2:19.3.0-4775.g7f24e589.el9.x86_64 2024-09-14T08:31:10.472 INFO:teuthology.orchestra.run.smithi164.stdout: ceph-mgr-modules-core-2:19.3.0-4775.g7f24e589.el9.noarch 2024-09-14T08:31:10.472 INFO:teuthology.orchestra.run.smithi164.stdout: ceph-mon-2:19.3.0-4775.g7f24e589.el9.x86_64 2024-09-14T08:31:10.473 INFO:teuthology.orchestra.run.smithi164.stdout: ceph-osd-2:19.3.0-4775.g7f24e589.el9.x86_64 2024-09-14T08:31:10.473 INFO:teuthology.orchestra.run.smithi164.stdout: lua-5.4.4-4.el9.x86_64 2024-09-14T08:31:10.473 INFO:teuthology.orchestra.run.smithi164.stdout: lua-devel-5.4.4-4.el9.x86_64 2024-09-14T08:31:10.473 INFO:teuthology.orchestra.run.smithi164.stdout: luarocks-3.9.2-5.el9.noarch 2024-09-14T08:31:10.473 INFO:teuthology.orchestra.run.smithi164.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2024-09-14T08:31:10.473 INFO:teuthology.orchestra.run.smithi164.stdout: python3-cffi-1.14.5-5.el9.x86_64 2024-09-14T08:31:10.473 INFO:teuthology.orchestra.run.smithi164.stdout: python3-cheroot-8.6.0-4.el9.noarch 2024-09-14T08:31:10.473 INFO:teuthology.orchestra.run.smithi164.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2024-09-14T08:31:10.473 INFO:teuthology.orchestra.run.smithi164.stdout: python3-cryptography-36.0.1-4.el9.x86_64 2024-09-14T08:31:10.473 INFO:teuthology.orchestra.run.smithi164.stdout: python3-jaraco-8.2.1-3.el9.noarch 2024-09-14T08:31:10.473 INFO:teuthology.orchestra.run.smithi164.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2024-09-14T08:31:10.473 INFO:teuthology.orchestra.run.smithi164.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2024-09-14T08:31:10.473 INFO:teuthology.orchestra.run.smithi164.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2024-09-14T08:31:10.473 INFO:teuthology.orchestra.run.smithi164.stdout: python3-jaraco-text-3.2.0-6.el9.noarch 2024-09-14T08:31:10.474 INFO:teuthology.orchestra.run.smithi164.stdout: python3-logutils-0.3.5-21.el9.noarch 2024-09-14T08:31:10.474 INFO:teuthology.orchestra.run.smithi164.stdout: python3-mako-1.1.4-6.el9.noarch 2024-09-14T08:31:10.474 INFO:teuthology.orchestra.run.smithi164.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2024-09-14T08:31:10.474 INFO:teuthology.orchestra.run.smithi164.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2024-09-14T08:31:10.474 INFO:teuthology.orchestra.run.smithi164.stdout: python3-packaging-20.9-5.el9.noarch 2024-09-14T08:31:10.474 INFO:teuthology.orchestra.run.smithi164.stdout: python3-pecan-1.4.2-3.el9.noarch 2024-09-14T08:31:10.474 INFO:teuthology.orchestra.run.smithi164.stdout: python3-ply-3.11-14.el9.noarch 2024-09-14T08:31:10.474 INFO:teuthology.orchestra.run.smithi164.stdout: python3-portend-3.1.0-2.el9.noarch 2024-09-14T08:31:10.474 INFO:teuthology.orchestra.run.smithi164.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2024-09-14T08:31:10.474 INFO:teuthology.orchestra.run.smithi164.stdout: python3-pycparser-2.20-6.el9.noarch 2024-09-14T08:31:10.474 INFO:teuthology.orchestra.run.smithi164.stdout: python3-pytz-2021.1-5.el9.noarch 2024-09-14T08:31:10.474 INFO:teuthology.orchestra.run.smithi164.stdout: python3-requests-2.25.1-8.el9.noarch 2024-09-14T08:31:10.474 INFO:teuthology.orchestra.run.smithi164.stdout: python3-tempora-5.0.0-2.el9.noarch 2024-09-14T08:31:10.474 INFO:teuthology.orchestra.run.smithi164.stdout: python3-toml-0.10.2-6.el9.noarch 2024-09-14T08:31:10.475 INFO:teuthology.orchestra.run.smithi164.stdout: python3-urllib3-1.26.5-6.el9.noarch 2024-09-14T08:31:10.475 INFO:teuthology.orchestra.run.smithi164.stdout: python3-webob-1.8.8-2.el9.noarch 2024-09-14T08:31:10.475 INFO:teuthology.orchestra.run.smithi164.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2024-09-14T08:31:10.475 INFO:teuthology.orchestra.run.smithi164.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2024-09-14T08:31:10.475 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:31:10.475 INFO:teuthology.orchestra.run.smithi164.stdout:Complete! 2024-09-14T08:31:10.588 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: ceph-2:19.3.0-4775.g7f24e589.el9.x86_64 37/37 2024-09-14T08:31:10.589 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : ceph-2:19.3.0-4775.g7f24e589.el9.x86_64 1/37 2024-09-14T08:31:10.589 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : ceph-mds-2:19.3.0-4775.g7f24e589.el9.x86_64 2/37 2024-09-14T08:31:10.589 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : ceph-mgr-2:19.3.0-4775.g7f24e589.el9.x86_64 3/37 2024-09-14T08:31:10.589 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : ceph-mon-2:19.3.0-4775.g7f24e589.el9.x86_64 4/37 2024-09-14T08:31:10.589 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : ceph-osd-2:19.3.0-4775.g7f24e589.el9.x86_64 5/37 2024-09-14T08:31:10.589 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : ceph-mgr-modules-core-2:19.3.0-4775.g7f24e589.el9. 6/37 2024-09-14T08:31:10.589 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 7/37 2024-09-14T08:31:10.589 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-cryptography-36.0.1-4.el9.x86_64 8/37 2024-09-14T08:31:10.589 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-ply-3.11-14.el9.noarch 9/37 2024-09-14T08:31:10.589 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 10/37 2024-09-14T08:31:10.589 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-requests-2.25.1-8.el9.noarch 11/37 2024-09-14T08:31:10.589 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-urllib3-1.26.5-6.el9.noarch 12/37 2024-09-14T08:31:10.589 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : lua-5.4.4-4.el9.x86_64 13/37 2024-09-14T08:31:10.590 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 14/37 2024-09-14T08:31:10.590 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 15/37 2024-09-14T08:31:10.590 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-packaging-20.9-5.el9.noarch 16/37 2024-09-14T08:31:10.590 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-pytz-2021.1-5.el9.noarch 17/37 2024-09-14T08:31:10.590 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 18/37 2024-09-14T08:31:10.590 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : lua-devel-5.4.4-4.el9.x86_64 19/37 2024-09-14T08:31:10.590 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : luarocks-3.9.2-5.el9.noarch 20/37 2024-09-14T08:31:10.590 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 21/37 2024-09-14T08:31:10.590 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-cheroot-8.6.0-4.el9.noarch 22/37 2024-09-14T08:31:10.590 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 23/37 2024-09-14T08:31:10.590 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 24/37 2024-09-14T08:31:10.590 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 25/37 2024-09-14T08:31:10.590 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 26/37 2024-09-14T08:31:10.590 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 27/37 2024-09-14T08:31:10.591 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-jaraco-text-3.2.0-6.el9.noarch 28/37 2024-09-14T08:31:10.591 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 29/37 2024-09-14T08:31:10.591 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 30/37 2024-09-14T08:31:10.591 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 31/37 2024-09-14T08:31:10.591 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 32/37 2024-09-14T08:31:10.591 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 33/37 2024-09-14T08:31:10.591 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 34/37 2024-09-14T08:31:10.591 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-webob-1.8.8-2.el9.noarch 35/37 2024-09-14T08:31:10.591 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 36/37 2024-09-14T08:31:10.857 DEBUG:teuthology.orchestra.run.smithi164:> sudo yum -y install ceph-base 2024-09-14T08:31:11.439 INFO:teuthology.orchestra.run.smithi164.stdout:Last metadata expiration check: 0:00:55 ago on Sat 14 Sep 2024 08:30:16 AM UTC. 2024-09-14T08:31:11.503 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 37/37 2024-09-14T08:31:11.503 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:31:11.503 INFO:teuthology.orchestra.run.smithi135.stdout:Installed: 2024-09-14T08:31:11.503 INFO:teuthology.orchestra.run.smithi135.stdout: ceph-2:19.3.0-4775.g7f24e589.el9.x86_64 2024-09-14T08:31:11.503 INFO:teuthology.orchestra.run.smithi135.stdout: ceph-mds-2:19.3.0-4775.g7f24e589.el9.x86_64 2024-09-14T08:31:11.503 INFO:teuthology.orchestra.run.smithi135.stdout: ceph-mgr-2:19.3.0-4775.g7f24e589.el9.x86_64 2024-09-14T08:31:11.503 INFO:teuthology.orchestra.run.smithi135.stdout: ceph-mgr-modules-core-2:19.3.0-4775.g7f24e589.el9.noarch 2024-09-14T08:31:11.503 INFO:teuthology.orchestra.run.smithi135.stdout: ceph-mon-2:19.3.0-4775.g7f24e589.el9.x86_64 2024-09-14T08:31:11.503 INFO:teuthology.orchestra.run.smithi135.stdout: ceph-osd-2:19.3.0-4775.g7f24e589.el9.x86_64 2024-09-14T08:31:11.503 INFO:teuthology.orchestra.run.smithi135.stdout: lua-5.4.4-4.el9.x86_64 2024-09-14T08:31:11.504 INFO:teuthology.orchestra.run.smithi135.stdout: lua-devel-5.4.4-4.el9.x86_64 2024-09-14T08:31:11.504 INFO:teuthology.orchestra.run.smithi135.stdout: luarocks-3.9.2-5.el9.noarch 2024-09-14T08:31:11.504 INFO:teuthology.orchestra.run.smithi135.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2024-09-14T08:31:11.504 INFO:teuthology.orchestra.run.smithi135.stdout: python3-cffi-1.14.5-5.el9.x86_64 2024-09-14T08:31:11.504 INFO:teuthology.orchestra.run.smithi135.stdout: python3-cheroot-8.6.0-4.el9.noarch 2024-09-14T08:31:11.504 INFO:teuthology.orchestra.run.smithi135.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2024-09-14T08:31:11.504 INFO:teuthology.orchestra.run.smithi135.stdout: python3-cryptography-36.0.1-4.el9.x86_64 2024-09-14T08:31:11.504 INFO:teuthology.orchestra.run.smithi135.stdout: python3-jaraco-8.2.1-3.el9.noarch 2024-09-14T08:31:11.504 INFO:teuthology.orchestra.run.smithi135.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2024-09-14T08:31:11.504 INFO:teuthology.orchestra.run.smithi135.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2024-09-14T08:31:11.504 INFO:teuthology.orchestra.run.smithi135.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2024-09-14T08:31:11.504 INFO:teuthology.orchestra.run.smithi135.stdout: python3-jaraco-text-3.2.0-6.el9.noarch 2024-09-14T08:31:11.504 INFO:teuthology.orchestra.run.smithi135.stdout: python3-logutils-0.3.5-21.el9.noarch 2024-09-14T08:31:11.505 INFO:teuthology.orchestra.run.smithi135.stdout: python3-mako-1.1.4-6.el9.noarch 2024-09-14T08:31:11.505 INFO:teuthology.orchestra.run.smithi135.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2024-09-14T08:31:11.505 INFO:teuthology.orchestra.run.smithi135.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2024-09-14T08:31:11.505 INFO:teuthology.orchestra.run.smithi135.stdout: python3-packaging-20.9-5.el9.noarch 2024-09-14T08:31:11.505 INFO:teuthology.orchestra.run.smithi135.stdout: python3-pecan-1.4.2-3.el9.noarch 2024-09-14T08:31:11.505 INFO:teuthology.orchestra.run.smithi135.stdout: python3-ply-3.11-14.el9.noarch 2024-09-14T08:31:11.505 INFO:teuthology.orchestra.run.smithi135.stdout: python3-portend-3.1.0-2.el9.noarch 2024-09-14T08:31:11.505 INFO:teuthology.orchestra.run.smithi135.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2024-09-14T08:31:11.505 INFO:teuthology.orchestra.run.smithi135.stdout: python3-pycparser-2.20-6.el9.noarch 2024-09-14T08:31:11.505 INFO:teuthology.orchestra.run.smithi135.stdout: python3-pytz-2021.1-5.el9.noarch 2024-09-14T08:31:11.505 INFO:teuthology.orchestra.run.smithi135.stdout: python3-requests-2.25.1-8.el9.noarch 2024-09-14T08:31:11.505 INFO:teuthology.orchestra.run.smithi135.stdout: python3-tempora-5.0.0-2.el9.noarch 2024-09-14T08:31:11.505 INFO:teuthology.orchestra.run.smithi135.stdout: python3-toml-0.10.2-6.el9.noarch 2024-09-14T08:31:11.506 INFO:teuthology.orchestra.run.smithi135.stdout: python3-urllib3-1.26.5-6.el9.noarch 2024-09-14T08:31:11.506 INFO:teuthology.orchestra.run.smithi135.stdout: python3-webob-1.8.8-2.el9.noarch 2024-09-14T08:31:11.506 INFO:teuthology.orchestra.run.smithi135.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2024-09-14T08:31:11.506 INFO:teuthology.orchestra.run.smithi135.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2024-09-14T08:31:11.506 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:31:11.506 INFO:teuthology.orchestra.run.smithi135.stdout:Complete! 2024-09-14T08:31:11.531 INFO:teuthology.orchestra.run.smithi164.stdout:Package ceph-base-2:19.3.0-4775.g7f24e589.el9.x86_64 is already installed. 2024-09-14T08:31:11.573 INFO:teuthology.orchestra.run.smithi164.stdout:Dependencies resolved. 2024-09-14T08:31:11.574 INFO:teuthology.orchestra.run.smithi164.stdout:Nothing to do. 2024-09-14T08:31:11.574 INFO:teuthology.orchestra.run.smithi164.stdout:Complete! 2024-09-14T08:31:11.648 DEBUG:teuthology.orchestra.run.smithi164:> sudo yum -y install cephadm 2024-09-14T08:31:11.864 DEBUG:teuthology.orchestra.run.smithi135:> sudo yum -y install ceph-base 2024-09-14T08:31:12.211 INFO:teuthology.orchestra.run.smithi164.stdout:Last metadata expiration check: 0:00:56 ago on Sat 14 Sep 2024 08:30:16 AM UTC. 2024-09-14T08:31:12.344 INFO:teuthology.orchestra.run.smithi164.stdout:Dependencies resolved. 2024-09-14T08:31:12.345 INFO:teuthology.orchestra.run.smithi164.stdout:================================================================================ 2024-09-14T08:31:12.345 INFO:teuthology.orchestra.run.smithi164.stdout: Package Arch Version Repository Size 2024-09-14T08:31:12.345 INFO:teuthology.orchestra.run.smithi164.stdout:================================================================================ 2024-09-14T08:31:12.345 INFO:teuthology.orchestra.run.smithi164.stdout:Installing: 2024-09-14T08:31:12.345 INFO:teuthology.orchestra.run.smithi164.stdout: cephadm noarch 2:19.3.0-4775.g7f24e589.el9 ceph-noarch 783 k 2024-09-14T08:31:12.345 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:31:12.345 INFO:teuthology.orchestra.run.smithi164.stdout:Transaction Summary 2024-09-14T08:31:12.345 INFO:teuthology.orchestra.run.smithi164.stdout:================================================================================ 2024-09-14T08:31:12.345 INFO:teuthology.orchestra.run.smithi164.stdout:Install 1 Package 2024-09-14T08:31:12.345 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:31:12.346 INFO:teuthology.orchestra.run.smithi164.stdout:Total download size: 783 k 2024-09-14T08:31:12.346 INFO:teuthology.orchestra.run.smithi164.stdout:Installed size: 789 k 2024-09-14T08:31:12.346 INFO:teuthology.orchestra.run.smithi164.stdout:Downloading Packages: 2024-09-14T08:31:12.443 INFO:teuthology.orchestra.run.smithi135.stdout:Last metadata expiration check: 0:00:55 ago on Sat 14 Sep 2024 08:30:17 AM UTC. 2024-09-14T08:31:12.534 INFO:teuthology.orchestra.run.smithi135.stdout:Package ceph-base-2:19.3.0-4775.g7f24e589.el9.x86_64 is already installed. 2024-09-14T08:31:12.576 INFO:teuthology.orchestra.run.smithi135.stdout:Dependencies resolved. 2024-09-14T08:31:12.576 INFO:teuthology.orchestra.run.smithi135.stdout:Nothing to do. 2024-09-14T08:31:12.577 INFO:teuthology.orchestra.run.smithi135.stdout:Complete! 2024-09-14T08:31:12.603 INFO:teuthology.orchestra.run.smithi164.stdout:cephadm-19.3.0-4775.g7f24e589.el9.noarch.rpm 3.0 MB/s | 783 kB 00:00 2024-09-14T08:31:12.603 INFO:teuthology.orchestra.run.smithi164.stdout:-------------------------------------------------------------------------------- 2024-09-14T08:31:12.604 INFO:teuthology.orchestra.run.smithi164.stdout:Total 2.9 MB/s | 783 kB 00:00 2024-09-14T08:31:12.604 INFO:teuthology.orchestra.run.smithi164.stdout:Running transaction check 2024-09-14T08:31:12.609 INFO:teuthology.orchestra.run.smithi164.stdout:Transaction check succeeded. 2024-09-14T08:31:12.609 INFO:teuthology.orchestra.run.smithi164.stdout:Running transaction test 2024-09-14T08:31:12.615 INFO:teuthology.orchestra.run.smithi164.stdout:Transaction test succeeded. 2024-09-14T08:31:12.616 INFO:teuthology.orchestra.run.smithi164.stdout:Running transaction 2024-09-14T08:31:12.639 DEBUG:teuthology.orchestra.run.smithi135:> sudo yum -y install cephadm 2024-09-14T08:31:12.694 INFO:teuthology.orchestra.run.smithi164.stdout: Preparing : 1/1 2024-09-14T08:31:13.102 INFO:teuthology.orchestra.run.smithi164.stdout: Running scriptlet: cephadm-2:19.3.0-4775.g7f24e589.el9.noarch 1/1 2024-09-14T08:31:13.190 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : cephadm-2:19.3.0-4775.g7f24e589.el9.noarch 1/1 2024-09-14T08:31:13.205 INFO:teuthology.orchestra.run.smithi135.stdout:Last metadata expiration check: 0:00:56 ago on Sat 14 Sep 2024 08:30:17 AM UTC. 2024-09-14T08:31:13.339 INFO:teuthology.orchestra.run.smithi135.stdout:Dependencies resolved. 2024-09-14T08:31:13.340 INFO:teuthology.orchestra.run.smithi135.stdout:================================================================================ 2024-09-14T08:31:13.340 INFO:teuthology.orchestra.run.smithi135.stdout: Package Arch Version Repository Size 2024-09-14T08:31:13.340 INFO:teuthology.orchestra.run.smithi135.stdout:================================================================================ 2024-09-14T08:31:13.340 INFO:teuthology.orchestra.run.smithi135.stdout:Installing: 2024-09-14T08:31:13.340 INFO:teuthology.orchestra.run.smithi135.stdout: cephadm noarch 2:19.3.0-4775.g7f24e589.el9 ceph-noarch 783 k 2024-09-14T08:31:13.340 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:31:13.340 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction Summary 2024-09-14T08:31:13.340 INFO:teuthology.orchestra.run.smithi135.stdout:================================================================================ 2024-09-14T08:31:13.340 INFO:teuthology.orchestra.run.smithi135.stdout:Install 1 Package 2024-09-14T08:31:13.340 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:31:13.341 INFO:teuthology.orchestra.run.smithi135.stdout:Total download size: 783 k 2024-09-14T08:31:13.341 INFO:teuthology.orchestra.run.smithi135.stdout:Installed size: 789 k 2024-09-14T08:31:13.341 INFO:teuthology.orchestra.run.smithi135.stdout:Downloading Packages: 2024-09-14T08:31:13.549 INFO:teuthology.orchestra.run.smithi164.stdout: Running scriptlet: cephadm-2:19.3.0-4775.g7f24e589.el9.noarch 1/1 2024-09-14T08:31:13.608 INFO:teuthology.orchestra.run.smithi135.stdout:cephadm-19.3.0-4775.g7f24e589.el9.noarch.rpm 2.9 MB/s | 783 kB 00:00 2024-09-14T08:31:13.608 INFO:teuthology.orchestra.run.smithi135.stdout:-------------------------------------------------------------------------------- 2024-09-14T08:31:13.609 INFO:teuthology.orchestra.run.smithi135.stdout:Total 2.8 MB/s | 783 kB 00:00 2024-09-14T08:31:13.609 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction check 2024-09-14T08:31:13.614 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction check succeeded. 2024-09-14T08:31:13.614 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction test 2024-09-14T08:31:13.620 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction test succeeded. 2024-09-14T08:31:13.621 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction 2024-09-14T08:31:13.696 INFO:teuthology.orchestra.run.smithi135.stdout: Preparing : 1/1 2024-09-14T08:31:13.807 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : cephadm-2:19.3.0-4775.g7f24e589.el9.noarch 1/1 2024-09-14T08:31:13.807 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:31:13.807 INFO:teuthology.orchestra.run.smithi164.stdout:Installed: 2024-09-14T08:31:13.807 INFO:teuthology.orchestra.run.smithi164.stdout: cephadm-2:19.3.0-4775.g7f24e589.el9.noarch 2024-09-14T08:31:13.807 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:31:13.807 INFO:teuthology.orchestra.run.smithi164.stdout:Complete! 2024-09-14T08:31:14.007 DEBUG:teuthology.orchestra.run.smithi164:> sudo yum -y install ceph-immutable-object-cache 2024-09-14T08:31:14.174 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: cephadm-2:19.3.0-4775.g7f24e589.el9.noarch 1/1 2024-09-14T08:31:14.260 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : cephadm-2:19.3.0-4775.g7f24e589.el9.noarch 1/1 2024-09-14T08:31:14.583 INFO:teuthology.orchestra.run.smithi164.stdout:Last metadata expiration check: 0:00:58 ago on Sat 14 Sep 2024 08:30:16 AM UTC. 2024-09-14T08:31:14.622 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: cephadm-2:19.3.0-4775.g7f24e589.el9.noarch 1/1 2024-09-14T08:31:14.718 INFO:teuthology.orchestra.run.smithi164.stdout:Dependencies resolved. 2024-09-14T08:31:14.718 INFO:teuthology.orchestra.run.smithi164.stdout:================================================================================ 2024-09-14T08:31:14.719 INFO:teuthology.orchestra.run.smithi164.stdout: Package Arch Version Repo Size 2024-09-14T08:31:14.719 INFO:teuthology.orchestra.run.smithi164.stdout:================================================================================ 2024-09-14T08:31:14.719 INFO:teuthology.orchestra.run.smithi164.stdout:Installing: 2024-09-14T08:31:14.719 INFO:teuthology.orchestra.run.smithi164.stdout: ceph-immutable-object-cache x86_64 2:19.3.0-4775.g7f24e589.el9 ceph 145 k 2024-09-14T08:31:14.719 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:31:14.719 INFO:teuthology.orchestra.run.smithi164.stdout:Transaction Summary 2024-09-14T08:31:14.719 INFO:teuthology.orchestra.run.smithi164.stdout:================================================================================ 2024-09-14T08:31:14.719 INFO:teuthology.orchestra.run.smithi164.stdout:Install 1 Package 2024-09-14T08:31:14.720 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:31:14.720 INFO:teuthology.orchestra.run.smithi164.stdout:Total download size: 145 k 2024-09-14T08:31:14.720 INFO:teuthology.orchestra.run.smithi164.stdout:Installed size: 432 k 2024-09-14T08:31:14.720 INFO:teuthology.orchestra.run.smithi164.stdout:Downloading Packages: 2024-09-14T08:31:14.872 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : cephadm-2:19.3.0-4775.g7f24e589.el9.noarch 1/1 2024-09-14T08:31:14.872 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:31:14.873 INFO:teuthology.orchestra.run.smithi135.stdout:Installed: 2024-09-14T08:31:14.873 INFO:teuthology.orchestra.run.smithi135.stdout: cephadm-2:19.3.0-4775.g7f24e589.el9.noarch 2024-09-14T08:31:14.873 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:31:14.873 INFO:teuthology.orchestra.run.smithi135.stdout:Complete! 2024-09-14T08:31:14.952 INFO:teuthology.orchestra.run.smithi164.stdout:ceph-immutable-object-cache-19.3.0-4775.g7f24e5 624 kB/s | 145 kB 00:00 2024-09-14T08:31:14.952 INFO:teuthology.orchestra.run.smithi164.stdout:-------------------------------------------------------------------------------- 2024-09-14T08:31:14.953 INFO:teuthology.orchestra.run.smithi164.stdout:Total 619 kB/s | 145 kB 00:00 2024-09-14T08:31:14.953 INFO:teuthology.orchestra.run.smithi164.stdout:Running transaction check 2024-09-14T08:31:14.961 INFO:teuthology.orchestra.run.smithi164.stdout:Transaction check succeeded. 2024-09-14T08:31:14.961 INFO:teuthology.orchestra.run.smithi164.stdout:Running transaction test 2024-09-14T08:31:15.017 INFO:teuthology.orchestra.run.smithi164.stdout:Transaction test succeeded. 2024-09-14T08:31:15.018 INFO:teuthology.orchestra.run.smithi164.stdout:Running transaction 2024-09-14T08:31:15.079 DEBUG:teuthology.orchestra.run.smithi135:> sudo yum -y install ceph-immutable-object-cache 2024-09-14T08:31:15.182 INFO:teuthology.orchestra.run.smithi164.stdout: Preparing : 1/1 2024-09-14T08:31:15.270 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : ceph-immutable-object-cache-2:19.3.0-4775.g7f24e589. 1/1 2024-09-14T08:31:15.295 INFO:teuthology.orchestra.run.smithi164.stdout: Running scriptlet: ceph-immutable-object-cache-2:19.3.0-4775.g7f24e589. 1/1 2024-09-14T08:31:15.295 INFO:teuthology.orchestra.run.smithi164.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-14T08:31:15.295 INFO:teuthology.orchestra.run.smithi164.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2024-09-14T08:31:15.295 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:31:15.662 INFO:teuthology.orchestra.run.smithi135.stdout:Last metadata expiration check: 0:00:58 ago on Sat 14 Sep 2024 08:30:17 AM UTC. 2024-09-14T08:31:15.802 INFO:teuthology.orchestra.run.smithi135.stdout:Dependencies resolved. 2024-09-14T08:31:15.802 INFO:teuthology.orchestra.run.smithi135.stdout:================================================================================ 2024-09-14T08:31:15.802 INFO:teuthology.orchestra.run.smithi135.stdout: Package Arch Version Repo Size 2024-09-14T08:31:15.803 INFO:teuthology.orchestra.run.smithi135.stdout:================================================================================ 2024-09-14T08:31:15.803 INFO:teuthology.orchestra.run.smithi135.stdout:Installing: 2024-09-14T08:31:15.803 INFO:teuthology.orchestra.run.smithi135.stdout: ceph-immutable-object-cache x86_64 2:19.3.0-4775.g7f24e589.el9 ceph 145 k 2024-09-14T08:31:15.803 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:31:15.803 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction Summary 2024-09-14T08:31:15.803 INFO:teuthology.orchestra.run.smithi135.stdout:================================================================================ 2024-09-14T08:31:15.803 INFO:teuthology.orchestra.run.smithi135.stdout:Install 1 Package 2024-09-14T08:31:15.803 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:31:15.803 INFO:teuthology.orchestra.run.smithi135.stdout:Total download size: 145 k 2024-09-14T08:31:15.803 INFO:teuthology.orchestra.run.smithi135.stdout:Installed size: 432 k 2024-09-14T08:31:15.803 INFO:teuthology.orchestra.run.smithi135.stdout:Downloading Packages: 2024-09-14T08:31:16.005 INFO:teuthology.orchestra.run.smithi135.stdout:ceph-immutable-object-cache-19.3.0-4775.g7f24e5 719 kB/s | 145 kB 00:00 2024-09-14T08:31:16.005 INFO:teuthology.orchestra.run.smithi135.stdout:-------------------------------------------------------------------------------- 2024-09-14T08:31:16.006 INFO:teuthology.orchestra.run.smithi135.stdout:Total 713 kB/s | 145 kB 00:00 2024-09-14T08:31:16.006 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction check 2024-09-14T08:31:16.013 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction check succeeded. 2024-09-14T08:31:16.013 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction test 2024-09-14T08:31:16.071 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction test succeeded. 2024-09-14T08:31:16.072 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction 2024-09-14T08:31:16.220 INFO:teuthology.orchestra.run.smithi135.stdout: Preparing : 1/1 2024-09-14T08:31:16.248 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : ceph-immutable-object-cache-2:19.3.0-4775.g7f24e589. 1/1 2024-09-14T08:31:16.248 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:31:16.248 INFO:teuthology.orchestra.run.smithi164.stdout:Installed: 2024-09-14T08:31:16.248 INFO:teuthology.orchestra.run.smithi164.stdout: ceph-immutable-object-cache-2:19.3.0-4775.g7f24e589.el9.x86_64 2024-09-14T08:31:16.249 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:31:16.249 INFO:teuthology.orchestra.run.smithi164.stdout:Complete! 2024-09-14T08:31:16.289 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : ceph-immutable-object-cache-2:19.3.0-4775.g7f24e589. 1/1 2024-09-14T08:31:16.314 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: ceph-immutable-object-cache-2:19.3.0-4775.g7f24e589. 1/1 2024-09-14T08:31:16.314 INFO:teuthology.orchestra.run.smithi135.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-14T08:31:16.314 INFO:teuthology.orchestra.run.smithi135.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2024-09-14T08:31:16.314 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:31:16.475 DEBUG:teuthology.orchestra.run.smithi164:> sudo yum -y install ceph-mgr 2024-09-14T08:31:17.043 INFO:teuthology.orchestra.run.smithi164.stdout:Last metadata expiration check: 0:01:01 ago on Sat 14 Sep 2024 08:30:16 AM UTC. 2024-09-14T08:31:17.136 INFO:teuthology.orchestra.run.smithi164.stdout:Package ceph-mgr-2:19.3.0-4775.g7f24e589.el9.x86_64 is already installed. 2024-09-14T08:31:17.180 INFO:teuthology.orchestra.run.smithi164.stdout:Dependencies resolved. 2024-09-14T08:31:17.180 INFO:teuthology.orchestra.run.smithi164.stdout:Nothing to do. 2024-09-14T08:31:17.180 INFO:teuthology.orchestra.run.smithi164.stdout:Complete! 2024-09-14T08:31:17.188 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : ceph-immutable-object-cache-2:19.3.0-4775.g7f24e589. 1/1 2024-09-14T08:31:17.189 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:31:17.189 INFO:teuthology.orchestra.run.smithi135.stdout:Installed: 2024-09-14T08:31:17.189 INFO:teuthology.orchestra.run.smithi135.stdout: ceph-immutable-object-cache-2:19.3.0-4775.g7f24e589.el9.x86_64 2024-09-14T08:31:17.189 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:31:17.189 INFO:teuthology.orchestra.run.smithi135.stdout:Complete! 2024-09-14T08:31:17.249 DEBUG:teuthology.orchestra.run.smithi164:> sudo yum -y install ceph-mgr-dashboard 2024-09-14T08:31:17.425 DEBUG:teuthology.orchestra.run.smithi135:> sudo yum -y install ceph-mgr 2024-09-14T08:31:17.810 INFO:teuthology.orchestra.run.smithi164.stdout:Last metadata expiration check: 0:01:01 ago on Sat 14 Sep 2024 08:30:16 AM UTC. 2024-09-14T08:31:17.943 INFO:teuthology.orchestra.run.smithi164.stdout:Dependencies resolved. 2024-09-14T08:31:17.944 INFO:teuthology.orchestra.run.smithi164.stdout:================================================================================ 2024-09-14T08:31:17.944 INFO:teuthology.orchestra.run.smithi164.stdout: Package Arch Version Repository Size 2024-09-14T08:31:17.945 INFO:teuthology.orchestra.run.smithi164.stdout:================================================================================ 2024-09-14T08:31:17.945 INFO:teuthology.orchestra.run.smithi164.stdout:Installing: 2024-09-14T08:31:17.945 INFO:teuthology.orchestra.run.smithi164.stdout: ceph-mgr-dashboard noarch 2:19.3.0-4775.g7f24e589.el9 ceph-noarch 14 M 2024-09-14T08:31:17.945 INFO:teuthology.orchestra.run.smithi164.stdout:Installing dependencies: 2024-09-14T08:31:17.945 INFO:teuthology.orchestra.run.smithi164.stdout: abseil-cpp x86_64 20211102.0-3.el9 epel 548 k 2024-09-14T08:31:17.945 INFO:teuthology.orchestra.run.smithi164.stdout: ceph-grafana-dashboards noarch 2:19.3.0-4775.g7f24e589.el9 ceph-noarch 34 k 2024-09-14T08:31:17.945 INFO:teuthology.orchestra.run.smithi164.stdout: ceph-prometheus-alerts noarch 2:19.3.0-4775.g7f24e589.el9 ceph-noarch 16 k 2024-09-14T08:31:17.945 INFO:teuthology.orchestra.run.smithi164.stdout: grpc-data noarch 1.46.7-10.el9 epel 19 k 2024-09-14T08:31:17.945 INFO:teuthology.orchestra.run.smithi164.stdout: protobuf x86_64 3.14.0-13.el9 appstream 1.0 M 2024-09-14T08:31:17.945 INFO:teuthology.orchestra.run.smithi164.stdout: protobuf-compiler x86_64 3.14.0-13.el9 crb 863 k 2024-09-14T08:31:17.945 INFO:teuthology.orchestra.run.smithi164.stdout: python3-grpcio x86_64 1.46.7-10.el9 epel 2.0 M 2024-09-14T08:31:17.945 INFO:teuthology.orchestra.run.smithi164.stdout: python3-grpcio-tools x86_64 1.46.7-10.el9 epel 144 k 2024-09-14T08:31:17.945 INFO:teuthology.orchestra.run.smithi164.stdout: python3-protobuf noarch 3.14.0-13.el9 appstream 269 k 2024-09-14T08:31:17.945 INFO:teuthology.orchestra.run.smithi164.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2024-09-14T08:31:17.946 INFO:teuthology.orchestra.run.smithi164.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2024-09-14T08:31:17.946 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:31:17.946 INFO:teuthology.orchestra.run.smithi164.stdout:Transaction Summary 2024-09-14T08:31:17.946 INFO:teuthology.orchestra.run.smithi164.stdout:================================================================================ 2024-09-14T08:31:17.946 INFO:teuthology.orchestra.run.smithi164.stdout:Install 12 Packages 2024-09-14T08:31:17.946 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:31:17.946 INFO:teuthology.orchestra.run.smithi164.stdout:Total download size: 19 M 2024-09-14T08:31:17.946 INFO:teuthology.orchestra.run.smithi164.stdout:Installed size: 159 M 2024-09-14T08:31:17.946 INFO:teuthology.orchestra.run.smithi164.stdout:Downloading Packages: 2024-09-14T08:31:18.004 INFO:teuthology.orchestra.run.smithi135.stdout:Last metadata expiration check: 0:01:01 ago on Sat 14 Sep 2024 08:30:17 AM UTC. 2024-09-14T08:31:18.099 INFO:teuthology.orchestra.run.smithi135.stdout:Package ceph-mgr-2:19.3.0-4775.g7f24e589.el9.x86_64 is already installed. 2024-09-14T08:31:18.142 INFO:teuthology.orchestra.run.smithi135.stdout:Dependencies resolved. 2024-09-14T08:31:18.143 INFO:teuthology.orchestra.run.smithi135.stdout:Nothing to do. 2024-09-14T08:31:18.143 INFO:teuthology.orchestra.run.smithi135.stdout:Complete! 2024-09-14T08:31:18.211 DEBUG:teuthology.orchestra.run.smithi135:> sudo yum -y install ceph-mgr-dashboard 2024-09-14T08:31:18.364 INFO:teuthology.orchestra.run.smithi164.stdout:(1/12): ceph-grafana-dashboards-19.3.0-4775.g7f 224 kB/s | 34 kB 00:00 2024-09-14T08:31:18.390 INFO:teuthology.orchestra.run.smithi164.stdout:(2/12): ceph-prometheus-alerts-19.3.0-4775.g7f2 92 kB/s | 16 kB 00:00 2024-09-14T08:31:18.582 INFO:teuthology.orchestra.run.smithi164.stdout:(3/12): python3-protobuf-3.14.0-13.el9.noarch.r 1.4 MB/s | 269 kB 00:00 2024-09-14T08:31:18.724 INFO:teuthology.orchestra.run.smithi164.stdout:(4/12): protobuf-3.14.0-13.el9.x86_64.rpm 2.8 MB/s | 1.0 MB 00:00 2024-09-14T08:31:18.766 INFO:teuthology.orchestra.run.smithi164.stdout:(5/12): protobuf-compiler-3.14.0-13.el9.x86_64. 4.6 MB/s | 863 kB 00:00 2024-09-14T08:31:18.786 INFO:teuthology.orchestra.run.smithi135.stdout:Last metadata expiration check: 0:01:01 ago on Sat 14 Sep 2024 08:30:17 AM UTC. 2024-09-14T08:31:18.808 INFO:teuthology.orchestra.run.smithi164.stdout:(6/12): abseil-cpp-20211102.0-3.el9.x86_64.rpm 6.4 MB/s | 548 kB 00:00 2024-09-14T08:31:18.825 INFO:teuthology.orchestra.run.smithi164.stdout:(7/12): grpc-data-1.46.7-10.el9.noarch.rpm 342 kB/s | 19 kB 00:00 2024-09-14T08:31:18.921 INFO:teuthology.orchestra.run.smithi135.stdout:Dependencies resolved. 2024-09-14T08:31:18.922 INFO:teuthology.orchestra.run.smithi135.stdout:================================================================================ 2024-09-14T08:31:18.922 INFO:teuthology.orchestra.run.smithi135.stdout: Package Arch Version Repository Size 2024-09-14T08:31:18.922 INFO:teuthology.orchestra.run.smithi135.stdout:================================================================================ 2024-09-14T08:31:18.922 INFO:teuthology.orchestra.run.smithi135.stdout:Installing: 2024-09-14T08:31:18.922 INFO:teuthology.orchestra.run.smithi135.stdout: ceph-mgr-dashboard noarch 2:19.3.0-4775.g7f24e589.el9 ceph-noarch 14 M 2024-09-14T08:31:18.922 INFO:teuthology.orchestra.run.smithi135.stdout:Installing dependencies: 2024-09-14T08:31:18.922 INFO:teuthology.orchestra.run.smithi135.stdout: abseil-cpp x86_64 20211102.0-3.el9 epel 548 k 2024-09-14T08:31:18.922 INFO:teuthology.orchestra.run.smithi135.stdout: ceph-grafana-dashboards noarch 2:19.3.0-4775.g7f24e589.el9 ceph-noarch 34 k 2024-09-14T08:31:18.923 INFO:teuthology.orchestra.run.smithi135.stdout: ceph-prometheus-alerts noarch 2:19.3.0-4775.g7f24e589.el9 ceph-noarch 16 k 2024-09-14T08:31:18.923 INFO:teuthology.orchestra.run.smithi135.stdout: grpc-data noarch 1.46.7-10.el9 epel 19 k 2024-09-14T08:31:18.923 INFO:teuthology.orchestra.run.smithi135.stdout: protobuf x86_64 3.14.0-13.el9 appstream 1.0 M 2024-09-14T08:31:18.923 INFO:teuthology.orchestra.run.smithi135.stdout: protobuf-compiler x86_64 3.14.0-13.el9 crb 863 k 2024-09-14T08:31:18.923 INFO:teuthology.orchestra.run.smithi135.stdout: python3-grpcio x86_64 1.46.7-10.el9 epel 2.0 M 2024-09-14T08:31:18.923 INFO:teuthology.orchestra.run.smithi135.stdout: python3-grpcio-tools x86_64 1.46.7-10.el9 epel 144 k 2024-09-14T08:31:18.923 INFO:teuthology.orchestra.run.smithi135.stdout: python3-protobuf noarch 3.14.0-13.el9 appstream 269 k 2024-09-14T08:31:18.923 INFO:teuthology.orchestra.run.smithi135.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2024-09-14T08:31:18.923 INFO:teuthology.orchestra.run.smithi135.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2024-09-14T08:31:18.923 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:31:18.923 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction Summary 2024-09-14T08:31:18.923 INFO:teuthology.orchestra.run.smithi135.stdout:================================================================================ 2024-09-14T08:31:18.923 INFO:teuthology.orchestra.run.smithi135.stdout:Install 12 Packages 2024-09-14T08:31:18.923 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:31:18.924 INFO:teuthology.orchestra.run.smithi135.stdout:Total download size: 19 M 2024-09-14T08:31:18.924 INFO:teuthology.orchestra.run.smithi135.stdout:Installed size: 159 M 2024-09-14T08:31:18.924 INFO:teuthology.orchestra.run.smithi135.stdout:Downloading Packages: 2024-09-14T08:31:18.983 INFO:teuthology.orchestra.run.smithi164.stdout:(8/12): ceph-mgr-dashboard-19.3.0-4775.g7f24e58 18 MB/s | 14 MB 00:00 2024-09-14T08:31:19.017 INFO:teuthology.orchestra.run.smithi164.stdout:(9/12): python3-grpcio-tools-1.46.7-10.el9.x86_ 753 kB/s | 144 kB 00:00 2024-09-14T08:31:19.042 INFO:teuthology.orchestra.run.smithi164.stdout:(10/12): python3-repoze-lru-0.7-16.el9.noarch.r 525 kB/s | 31 kB 00:00 2024-09-14T08:31:19.076 INFO:teuthology.orchestra.run.smithi164.stdout:(11/12): python3-routes-2.5.1-5.el9.noarch.rpm 3.2 MB/s | 188 kB 00:00 2024-09-14T08:31:19.126 INFO:teuthology.orchestra.run.smithi164.stdout:(12/12): python3-grpcio-1.46.7-10.el9.x86_64.rp 6.4 MB/s | 2.0 MB 00:00 2024-09-14T08:31:19.130 INFO:teuthology.orchestra.run.smithi164.stdout:-------------------------------------------------------------------------------- 2024-09-14T08:31:19.130 INFO:teuthology.orchestra.run.smithi164.stdout:Total 16 MB/s | 19 MB 00:01 2024-09-14T08:31:19.179 INFO:teuthology.orchestra.run.smithi164.stdout:Running transaction check 2024-09-14T08:31:19.198 INFO:teuthology.orchestra.run.smithi164.stdout:Transaction check succeeded. 2024-09-14T08:31:19.198 INFO:teuthology.orchestra.run.smithi164.stdout:Running transaction test 2024-09-14T08:31:19.477 INFO:teuthology.orchestra.run.smithi164.stdout:Transaction test succeeded. 2024-09-14T08:31:19.477 INFO:teuthology.orchestra.run.smithi164.stdout:Running transaction 2024-09-14T08:31:19.542 INFO:teuthology.orchestra.run.smithi135.stdout:(1/12): ceph-grafana-dashboards-19.3.0-4775.g7f 211 kB/s | 34 kB 00:00 2024-09-14T08:31:19.568 INFO:teuthology.orchestra.run.smithi135.stdout:(2/12): ceph-prometheus-alerts-19.3.0-4775.g7f2 88 kB/s | 16 kB 00:00 2024-09-14T08:31:19.709 INFO:teuthology.orchestra.run.smithi135.stdout:(3/12): python3-protobuf-3.14.0-13.el9.noarch.r 1.9 MB/s | 269 kB 00:00 2024-09-14T08:31:19.776 INFO:teuthology.orchestra.run.smithi135.stdout:(4/12): protobuf-3.14.0-13.el9.x86_64.rpm 4.3 MB/s | 1.0 MB 00:00 2024-09-14T08:31:19.827 INFO:teuthology.orchestra.run.smithi135.stdout:(5/12): abseil-cpp-20211102.0-3.el9.x86_64.rpm 11 MB/s | 548 kB 00:00 2024-09-14T08:31:19.843 INFO:teuthology.orchestra.run.smithi135.stdout:(6/12): grpc-data-1.46.7-10.el9.noarch.rpm 1.2 MB/s | 19 kB 00:00 2024-09-14T08:31:19.911 INFO:teuthology.orchestra.run.smithi135.stdout:(7/12): python3-grpcio-1.46.7-10.el9.x86_64.rpm 31 MB/s | 2.0 MB 00:00 2024-09-14T08:31:19.936 INFO:teuthology.orchestra.run.smithi135.stdout:(8/12): python3-grpcio-tools-1.46.7-10.el9.x86_ 5.7 MB/s | 144 kB 00:00 2024-09-14T08:31:19.961 INFO:teuthology.orchestra.run.smithi135.stdout:(9/12): python3-repoze-lru-0.7-16.el9.noarch.rp 1.2 MB/s | 31 kB 00:00 2024-09-14T08:31:19.986 INFO:teuthology.orchestra.run.smithi135.stdout:(10/12): python3-routes-2.5.1-5.el9.noarch.rpm 7.4 MB/s | 188 kB 00:00 2024-09-14T08:31:20.020 INFO:teuthology.orchestra.run.smithi135.stdout:(11/12): protobuf-compiler-3.14.0-13.el9.x86_64 2.7 MB/s | 863 kB 00:00 2024-09-14T08:31:20.072 INFO:teuthology.orchestra.run.smithi164.stdout: Preparing : 1/1 2024-09-14T08:31:20.143 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : protobuf-3.14.0-13.el9.x86_64 1/12 2024-09-14T08:31:20.154 INFO:teuthology.orchestra.run.smithi135.stdout:(12/12): ceph-mgr-dashboard-19.3.0-4775.g7f24e5 18 MB/s | 14 MB 00:00 2024-09-14T08:31:20.157 INFO:teuthology.orchestra.run.smithi135.stdout:-------------------------------------------------------------------------------- 2024-09-14T08:31:20.157 INFO:teuthology.orchestra.run.smithi135.stdout:Total 16 MB/s | 19 MB 00:01 2024-09-14T08:31:20.186 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : protobuf-compiler-3.14.0-13.el9.x86_64 2/12 2024-09-14T08:31:20.205 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction check 2024-09-14T08:31:20.224 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction check succeeded. 2024-09-14T08:31:20.224 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction test 2024-09-14T08:31:20.241 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 3/12 2024-09-14T08:31:20.281 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 4/12 2024-09-14T08:31:20.393 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : grpc-data-1.46.7-10.el9.noarch 5/12 2024-09-14T08:31:20.494 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction test succeeded. 2024-09-14T08:31:20.494 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction 2024-09-14T08:31:20.525 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : abseil-cpp-20211102.0-3.el9.x86_64 6/12 2024-09-14T08:31:20.618 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : python3-grpcio-1.46.7-10.el9.x86_64 7/12 2024-09-14T08:31:20.669 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : python3-protobuf-3.14.0-13.el9.noarch 8/12 2024-09-14T08:31:20.698 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : python3-grpcio-tools-1.46.7-10.el9.x86_64 9/12 2024-09-14T08:31:20.730 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : ceph-prometheus-alerts-2:19.3.0-4775.g7f24e589.el9 10/12 2024-09-14T08:31:21.102 INFO:teuthology.orchestra.run.smithi135.stdout: Preparing : 1/1 2024-09-14T08:31:21.188 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : protobuf-3.14.0-13.el9.x86_64 1/12 2024-09-14T08:31:21.232 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : protobuf-compiler-3.14.0-13.el9.x86_64 2/12 2024-09-14T08:31:21.278 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 3/12 2024-09-14T08:31:21.308 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 4/12 2024-09-14T08:31:21.403 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : grpc-data-1.46.7-10.el9.noarch 5/12 2024-09-14T08:31:21.527 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : abseil-cpp-20211102.0-3.el9.x86_64 6/12 2024-09-14T08:31:21.628 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-grpcio-1.46.7-10.el9.x86_64 7/12 2024-09-14T08:31:21.680 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-protobuf-3.14.0-13.el9.noarch 8/12 2024-09-14T08:31:21.716 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-grpcio-tools-1.46.7-10.el9.x86_64 9/12 2024-09-14T08:31:21.749 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : ceph-prometheus-alerts-2:19.3.0-4775.g7f24e589.el9 10/12 2024-09-14T08:31:23.924 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : ceph-grafana-dashboards-2:19.3.0-4775.g7f24e589.el 11/12 2024-09-14T08:31:23.976 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : ceph-mgr-dashboard-2:19.3.0-4775.g7f24e589.el9.noa 12/12 2024-09-14T08:31:24.930 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : ceph-grafana-dashboards-2:19.3.0-4775.g7f24e589.el 11/12 2024-09-14T08:31:24.986 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : ceph-mgr-dashboard-2:19.3.0-4775.g7f24e589.el9.noa 12/12 2024-09-14T08:31:26.003 INFO:teuthology.orchestra.run.smithi164.stdout: Running scriptlet: ceph-mgr-dashboard-2:19.3.0-4775.g7f24e589.el9.noa 12/12 2024-09-14T08:31:26.004 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : ceph-grafana-dashboards-2:19.3.0-4775.g7f24e589.el 1/12 2024-09-14T08:31:26.004 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : ceph-mgr-dashboard-2:19.3.0-4775.g7f24e589.el9.noa 2/12 2024-09-14T08:31:26.004 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : ceph-prometheus-alerts-2:19.3.0-4775.g7f24e589.el9 3/12 2024-09-14T08:31:26.004 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : protobuf-3.14.0-13.el9.x86_64 4/12 2024-09-14T08:31:26.004 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : python3-protobuf-3.14.0-13.el9.noarch 5/12 2024-09-14T08:31:26.004 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : protobuf-compiler-3.14.0-13.el9.x86_64 6/12 2024-09-14T08:31:26.004 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : abseil-cpp-20211102.0-3.el9.x86_64 7/12 2024-09-14T08:31:26.004 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : grpc-data-1.46.7-10.el9.noarch 8/12 2024-09-14T08:31:26.004 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : python3-grpcio-1.46.7-10.el9.x86_64 9/12 2024-09-14T08:31:26.004 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : python3-grpcio-tools-1.46.7-10.el9.x86_64 10/12 2024-09-14T08:31:26.004 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 11/12 2024-09-14T08:31:26.332 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 12/12 2024-09-14T08:31:26.332 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:31:26.332 INFO:teuthology.orchestra.run.smithi164.stdout:Installed: 2024-09-14T08:31:26.332 INFO:teuthology.orchestra.run.smithi164.stdout: abseil-cpp-20211102.0-3.el9.x86_64 2024-09-14T08:31:26.332 INFO:teuthology.orchestra.run.smithi164.stdout: ceph-grafana-dashboards-2:19.3.0-4775.g7f24e589.el9.noarch 2024-09-14T08:31:26.332 INFO:teuthology.orchestra.run.smithi164.stdout: ceph-mgr-dashboard-2:19.3.0-4775.g7f24e589.el9.noarch 2024-09-14T08:31:26.332 INFO:teuthology.orchestra.run.smithi164.stdout: ceph-prometheus-alerts-2:19.3.0-4775.g7f24e589.el9.noarch 2024-09-14T08:31:26.332 INFO:teuthology.orchestra.run.smithi164.stdout: grpc-data-1.46.7-10.el9.noarch 2024-09-14T08:31:26.332 INFO:teuthology.orchestra.run.smithi164.stdout: protobuf-3.14.0-13.el9.x86_64 2024-09-14T08:31:26.332 INFO:teuthology.orchestra.run.smithi164.stdout: protobuf-compiler-3.14.0-13.el9.x86_64 2024-09-14T08:31:26.332 INFO:teuthology.orchestra.run.smithi164.stdout: python3-grpcio-1.46.7-10.el9.x86_64 2024-09-14T08:31:26.333 INFO:teuthology.orchestra.run.smithi164.stdout: python3-grpcio-tools-1.46.7-10.el9.x86_64 2024-09-14T08:31:26.333 INFO:teuthology.orchestra.run.smithi164.stdout: python3-protobuf-3.14.0-13.el9.noarch 2024-09-14T08:31:26.333 INFO:teuthology.orchestra.run.smithi164.stdout: python3-repoze-lru-0.7-16.el9.noarch 2024-09-14T08:31:26.333 INFO:teuthology.orchestra.run.smithi164.stdout: python3-routes-2.5.1-5.el9.noarch 2024-09-14T08:31:26.333 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:31:26.333 INFO:teuthology.orchestra.run.smithi164.stdout:Complete! 2024-09-14T08:31:26.605 DEBUG:teuthology.orchestra.run.smithi164:> sudo yum -y install ceph-mgr-diskprediction-local 2024-09-14T08:31:26.930 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: ceph-mgr-dashboard-2:19.3.0-4775.g7f24e589.el9.noa 12/12 2024-09-14T08:31:26.931 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : ceph-grafana-dashboards-2:19.3.0-4775.g7f24e589.el 1/12 2024-09-14T08:31:26.931 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : ceph-mgr-dashboard-2:19.3.0-4775.g7f24e589.el9.noa 2/12 2024-09-14T08:31:26.931 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : ceph-prometheus-alerts-2:19.3.0-4775.g7f24e589.el9 3/12 2024-09-14T08:31:26.931 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : protobuf-3.14.0-13.el9.x86_64 4/12 2024-09-14T08:31:26.931 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-protobuf-3.14.0-13.el9.noarch 5/12 2024-09-14T08:31:26.931 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : protobuf-compiler-3.14.0-13.el9.x86_64 6/12 2024-09-14T08:31:26.931 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : abseil-cpp-20211102.0-3.el9.x86_64 7/12 2024-09-14T08:31:26.931 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : grpc-data-1.46.7-10.el9.noarch 8/12 2024-09-14T08:31:26.931 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-grpcio-1.46.7-10.el9.x86_64 9/12 2024-09-14T08:31:26.931 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-grpcio-tools-1.46.7-10.el9.x86_64 10/12 2024-09-14T08:31:26.932 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 11/12 2024-09-14T08:31:27.180 INFO:teuthology.orchestra.run.smithi164.stdout:Last metadata expiration check: 0:01:11 ago on Sat 14 Sep 2024 08:30:16 AM UTC. 2024-09-14T08:31:27.256 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 12/12 2024-09-14T08:31:27.256 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:31:27.256 INFO:teuthology.orchestra.run.smithi135.stdout:Installed: 2024-09-14T08:31:27.256 INFO:teuthology.orchestra.run.smithi135.stdout: abseil-cpp-20211102.0-3.el9.x86_64 2024-09-14T08:31:27.256 INFO:teuthology.orchestra.run.smithi135.stdout: ceph-grafana-dashboards-2:19.3.0-4775.g7f24e589.el9.noarch 2024-09-14T08:31:27.256 INFO:teuthology.orchestra.run.smithi135.stdout: ceph-mgr-dashboard-2:19.3.0-4775.g7f24e589.el9.noarch 2024-09-14T08:31:27.256 INFO:teuthology.orchestra.run.smithi135.stdout: ceph-prometheus-alerts-2:19.3.0-4775.g7f24e589.el9.noarch 2024-09-14T08:31:27.256 INFO:teuthology.orchestra.run.smithi135.stdout: grpc-data-1.46.7-10.el9.noarch 2024-09-14T08:31:27.256 INFO:teuthology.orchestra.run.smithi135.stdout: protobuf-3.14.0-13.el9.x86_64 2024-09-14T08:31:27.256 INFO:teuthology.orchestra.run.smithi135.stdout: protobuf-compiler-3.14.0-13.el9.x86_64 2024-09-14T08:31:27.257 INFO:teuthology.orchestra.run.smithi135.stdout: python3-grpcio-1.46.7-10.el9.x86_64 2024-09-14T08:31:27.257 INFO:teuthology.orchestra.run.smithi135.stdout: python3-grpcio-tools-1.46.7-10.el9.x86_64 2024-09-14T08:31:27.257 INFO:teuthology.orchestra.run.smithi135.stdout: python3-protobuf-3.14.0-13.el9.noarch 2024-09-14T08:31:27.257 INFO:teuthology.orchestra.run.smithi135.stdout: python3-repoze-lru-0.7-16.el9.noarch 2024-09-14T08:31:27.257 INFO:teuthology.orchestra.run.smithi135.stdout: python3-routes-2.5.1-5.el9.noarch 2024-09-14T08:31:27.257 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:31:27.257 INFO:teuthology.orchestra.run.smithi135.stdout:Complete! 2024-09-14T08:31:27.324 INFO:teuthology.orchestra.run.smithi164.stdout:Dependencies resolved. 2024-09-14T08:31:27.325 INFO:teuthology.orchestra.run.smithi164.stdout:======================================================================================= 2024-09-14T08:31:27.325 INFO:teuthology.orchestra.run.smithi164.stdout: Package Arch Version Repository Size 2024-09-14T08:31:27.325 INFO:teuthology.orchestra.run.smithi164.stdout:======================================================================================= 2024-09-14T08:31:27.325 INFO:teuthology.orchestra.run.smithi164.stdout:Installing: 2024-09-14T08:31:27.325 INFO:teuthology.orchestra.run.smithi164.stdout: ceph-mgr-diskprediction-local noarch 2:19.3.0-4775.g7f24e589.el9 ceph-noarch 7.4 M 2024-09-14T08:31:27.325 INFO:teuthology.orchestra.run.smithi164.stdout:Installing dependencies: 2024-09-14T08:31:27.325 INFO:teuthology.orchestra.run.smithi164.stdout: flexiblas x86_64 3.0.4-8.el9 appstream 33 k 2024-09-14T08:31:27.325 INFO:teuthology.orchestra.run.smithi164.stdout: flexiblas-netlib x86_64 3.0.4-8.el9 appstream 3.0 M 2024-09-14T08:31:27.325 INFO:teuthology.orchestra.run.smithi164.stdout: flexiblas-openblas-openmp x86_64 3.0.4-8.el9 appstream 18 k 2024-09-14T08:31:27.325 INFO:teuthology.orchestra.run.smithi164.stdout: libgfortran x86_64 11.5.0-2.el9 baseos 801 k 2024-09-14T08:31:27.326 INFO:teuthology.orchestra.run.smithi164.stdout: libquadmath x86_64 11.5.0-2.el9 baseos 191 k 2024-09-14T08:31:27.326 INFO:teuthology.orchestra.run.smithi164.stdout: openblas x86_64 0.3.26-2.el9 appstream 38 k 2024-09-14T08:31:27.326 INFO:teuthology.orchestra.run.smithi164.stdout: openblas-openmp x86_64 0.3.26-2.el9 appstream 4.9 M 2024-09-14T08:31:27.326 INFO:teuthology.orchestra.run.smithi164.stdout: python3-devel x86_64 3.9.19-8.el9 appstream 245 k 2024-09-14T08:31:27.326 INFO:teuthology.orchestra.run.smithi164.stdout: python3-numpy x86_64 1:1.20.1-5.el9 appstream 5.1 M 2024-09-14T08:31:27.326 INFO:teuthology.orchestra.run.smithi164.stdout: python3-numpy-f2py x86_64 1:1.20.1-5.el9 appstream 250 k 2024-09-14T08:31:27.326 INFO:teuthology.orchestra.run.smithi164.stdout: python3-scipy x86_64 1.6.2-8.el9 appstream 16 M 2024-09-14T08:31:27.326 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:31:27.326 INFO:teuthology.orchestra.run.smithi164.stdout:Transaction Summary 2024-09-14T08:31:27.326 INFO:teuthology.orchestra.run.smithi164.stdout:======================================================================================= 2024-09-14T08:31:27.326 INFO:teuthology.orchestra.run.smithi164.stdout:Install 12 Packages 2024-09-14T08:31:27.326 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:31:27.327 INFO:teuthology.orchestra.run.smithi164.stdout:Total download size: 38 M 2024-09-14T08:31:27.327 INFO:teuthology.orchestra.run.smithi164.stdout:Installed size: 212 M 2024-09-14T08:31:27.327 INFO:teuthology.orchestra.run.smithi164.stdout:Downloading Packages: 2024-09-14T08:31:27.526 DEBUG:teuthology.orchestra.run.smithi135:> sudo yum -y install ceph-mgr-diskprediction-local 2024-09-14T08:31:27.801 INFO:teuthology.orchestra.run.smithi164.stdout:(1/12): libquadmath-11.5.0-2.el9.x86_64.rpm 867 kB/s | 191 kB 00:00 2024-09-14T08:31:27.851 INFO:teuthology.orchestra.run.smithi164.stdout:(2/12): flexiblas-3.0.4-8.el9.x86_64.rpm 655 kB/s | 33 kB 00:00 2024-09-14T08:31:27.910 INFO:teuthology.orchestra.run.smithi164.stdout:(3/12): libgfortran-11.5.0-2.el9.x86_64.rpm 2.4 MB/s | 801 kB 00:00 2024-09-14T08:31:27.968 INFO:teuthology.orchestra.run.smithi164.stdout:(4/12): flexiblas-openblas-openmp-3.0.4-8.el9.x 312 kB/s | 18 kB 00:00 2024-09-14T08:31:28.019 INFO:teuthology.orchestra.run.smithi164.stdout:(5/12): openblas-0.3.26-2.el9.x86_64.rpm 770 kB/s | 38 kB 00:00 2024-09-14T08:31:28.113 INFO:teuthology.orchestra.run.smithi135.stdout:Last metadata expiration check: 0:01:11 ago on Sat 14 Sep 2024 08:30:17 AM UTC. 2024-09-14T08:31:28.119 INFO:teuthology.orchestra.run.smithi164.stdout:(6/12): ceph-mgr-diskprediction-local-19.3.0-47 14 MB/s | 7.4 MB 00:00 2024-09-14T08:31:28.256 INFO:teuthology.orchestra.run.smithi135.stdout:Dependencies resolved. 2024-09-14T08:31:28.257 INFO:teuthology.orchestra.run.smithi135.stdout:======================================================================================= 2024-09-14T08:31:28.257 INFO:teuthology.orchestra.run.smithi135.stdout: Package Arch Version Repository Size 2024-09-14T08:31:28.257 INFO:teuthology.orchestra.run.smithi135.stdout:======================================================================================= 2024-09-14T08:31:28.257 INFO:teuthology.orchestra.run.smithi135.stdout:Installing: 2024-09-14T08:31:28.257 INFO:teuthology.orchestra.run.smithi135.stdout: ceph-mgr-diskprediction-local noarch 2:19.3.0-4775.g7f24e589.el9 ceph-noarch 7.4 M 2024-09-14T08:31:28.257 INFO:teuthology.orchestra.run.smithi135.stdout:Installing dependencies: 2024-09-14T08:31:28.257 INFO:teuthology.orchestra.run.smithi135.stdout: flexiblas x86_64 3.0.4-8.el9 appstream 33 k 2024-09-14T08:31:28.258 INFO:teuthology.orchestra.run.smithi135.stdout: flexiblas-netlib x86_64 3.0.4-8.el9 appstream 3.0 M 2024-09-14T08:31:28.258 INFO:teuthology.orchestra.run.smithi135.stdout: flexiblas-openblas-openmp x86_64 3.0.4-8.el9 appstream 18 k 2024-09-14T08:31:28.258 INFO:teuthology.orchestra.run.smithi135.stdout: libgfortran x86_64 11.5.0-2.el9 baseos 801 k 2024-09-14T08:31:28.258 INFO:teuthology.orchestra.run.smithi135.stdout: libquadmath x86_64 11.5.0-2.el9 baseos 191 k 2024-09-14T08:31:28.258 INFO:teuthology.orchestra.run.smithi135.stdout: openblas x86_64 0.3.26-2.el9 appstream 38 k 2024-09-14T08:31:28.258 INFO:teuthology.orchestra.run.smithi135.stdout: openblas-openmp x86_64 0.3.26-2.el9 appstream 4.9 M 2024-09-14T08:31:28.258 INFO:teuthology.orchestra.run.smithi135.stdout: python3-devel x86_64 3.9.19-8.el9 appstream 245 k 2024-09-14T08:31:28.258 INFO:teuthology.orchestra.run.smithi135.stdout: python3-numpy x86_64 1:1.20.1-5.el9 appstream 5.1 M 2024-09-14T08:31:28.258 INFO:teuthology.orchestra.run.smithi135.stdout: python3-numpy-f2py x86_64 1:1.20.1-5.el9 appstream 250 k 2024-09-14T08:31:28.258 INFO:teuthology.orchestra.run.smithi135.stdout: python3-scipy x86_64 1.6.2-8.el9 appstream 16 M 2024-09-14T08:31:28.258 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:31:28.258 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction Summary 2024-09-14T08:31:28.258 INFO:teuthology.orchestra.run.smithi135.stdout:======================================================================================= 2024-09-14T08:31:28.258 INFO:teuthology.orchestra.run.smithi135.stdout:Install 12 Packages 2024-09-14T08:31:28.259 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:31:28.259 INFO:teuthology.orchestra.run.smithi135.stdout:Total download size: 38 M 2024-09-14T08:31:28.259 INFO:teuthology.orchestra.run.smithi135.stdout:Installed size: 212 M 2024-09-14T08:31:28.259 INFO:teuthology.orchestra.run.smithi135.stdout:Downloading Packages: 2024-09-14T08:31:28.353 INFO:teuthology.orchestra.run.smithi164.stdout:(7/12): python3-devel-3.9.19-8.el9.x86_64.rpm 1.0 MB/s | 245 kB 00:00 2024-09-14T08:31:28.694 INFO:teuthology.orchestra.run.smithi135.stdout:(1/12): libquadmath-11.5.0-2.el9.x86_64.rpm 909 kB/s | 191 kB 00:00 2024-09-14T08:31:28.794 INFO:teuthology.orchestra.run.smithi135.stdout:(2/12): flexiblas-3.0.4-8.el9.x86_64.rpm 328 kB/s | 33 kB 00:00 2024-09-14T08:31:28.836 INFO:teuthology.orchestra.run.smithi135.stdout:(3/12): libgfortran-11.5.0-2.el9.x86_64.rpm 2.2 MB/s | 801 kB 00:00 2024-09-14T08:31:28.912 INFO:teuthology.orchestra.run.smithi164.stdout:(8/12): flexiblas-netlib-3.0.4-8.el9.x86_64.rpm 2.8 MB/s | 3.0 MB 00:01 2024-09-14T08:31:28.928 INFO:teuthology.orchestra.run.smithi135.stdout:(4/12): flexiblas-openblas-openmp-3.0.4-8.el9.x 199 kB/s | 18 kB 00:00 2024-09-14T08:31:29.029 INFO:teuthology.orchestra.run.smithi135.stdout:(5/12): ceph-mgr-diskprediction-local-19.3.0-47 14 MB/s | 7.4 MB 00:00 2024-09-14T08:31:29.054 INFO:teuthology.orchestra.run.smithi164.stdout:(9/12): python3-numpy-f2py-1.20.1-5.el9.x86_64. 1.7 MB/s | 250 kB 00:00 2024-09-14T08:31:29.054 INFO:teuthology.orchestra.run.smithi135.stdout:(6/12): openblas-0.3.26-2.el9.x86_64.rpm 307 kB/s | 38 kB 00:00 2024-09-14T08:31:29.162 INFO:teuthology.orchestra.run.smithi135.stdout:(7/12): python3-devel-3.9.19-8.el9.x86_64.rpm 2.2 MB/s | 245 kB 00:00 2024-09-14T08:31:29.221 INFO:teuthology.orchestra.run.smithi135.stdout:(8/12): flexiblas-netlib-3.0.4-8.el9.x86_64.rpm 7.0 MB/s | 3.0 MB 00:00 2024-09-14T08:31:29.271 INFO:teuthology.orchestra.run.smithi135.stdout:(9/12): python3-numpy-f2py-1.20.1-5.el9.x86_64. 4.9 MB/s | 250 kB 00:00 2024-09-14T08:31:29.380 INFO:teuthology.orchestra.run.smithi135.stdout:(10/12): python3-numpy-1.20.1-5.el9.x86_64.rpm 23 MB/s | 5.1 MB 00:00 2024-09-14T08:31:29.456 INFO:teuthology.orchestra.run.smithi135.stdout:(11/12): openblas-openmp-0.3.26-2.el9.x86_64.rp 12 MB/s | 4.9 MB 00:00 2024-09-14T08:31:29.856 INFO:teuthology.orchestra.run.smithi135.stdout:(12/12): python3-scipy-1.6.2-8.el9.x86_64.rpm 28 MB/s | 16 MB 00:00 2024-09-14T08:31:29.857 INFO:teuthology.orchestra.run.smithi135.stdout:-------------------------------------------------------------------------------- 2024-09-14T08:31:29.857 INFO:teuthology.orchestra.run.smithi135.stdout:Total 24 MB/s | 38 MB 00:01 2024-09-14T08:31:30.004 INFO:teuthology.orchestra.run.smithi164.stdout:(10/12): openblas-openmp-0.3.26-2.el9.x86_64.rp 2.5 MB/s | 4.9 MB 00:01 2024-09-14T08:31:30.212 INFO:teuthology.orchestra.run.smithi164.stdout:(11/12): python3-numpy-1.20.1-5.el9.x86_64.rpm 2.7 MB/s | 5.1 MB 00:01 2024-09-14T08:31:30.213 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction check 2024-09-14T08:31:30.238 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction check succeeded. 2024-09-14T08:31:30.239 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction test 2024-09-14T08:31:30.673 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction test succeeded. 2024-09-14T08:31:30.673 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction 2024-09-14T08:31:31.433 INFO:teuthology.orchestra.run.smithi135.stdout: Preparing : 1/1 2024-09-14T08:31:31.512 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : flexiblas-3.0.4-8.el9.x86_64 1/12 2024-09-14T08:31:31.562 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : libquadmath-11.5.0-2.el9.x86_64 2/12 2024-09-14T08:31:31.664 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : libgfortran-11.5.0-2.el9.x86_64 3/12 2024-09-14T08:31:31.705 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-devel-3.9.19-8.el9.x86_64 4/12 2024-09-14T08:31:31.890 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : openblas-0.3.26-2.el9.x86_64 5/12 2024-09-14T08:31:31.912 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : openblas-openmp-0.3.26-2.el9.x86_64 6/12 2024-09-14T08:31:31.998 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 7/12 2024-09-14T08:31:32.650 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : flexiblas-netlib-3.0.4-8.el9.x86_64 8/12 2024-09-14T08:31:32.776 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-numpy-1:1.20.1-5.el9.x86_64 9/12 2024-09-14T08:31:34.024 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 10/12 2024-09-14T08:31:35.070 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-scipy-1.6.2-8.el9.x86_64 11/12 2024-09-14T08:31:35.095 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : ceph-mgr-diskprediction-local-2:19.3.0-4775.g7f24e 12/12 2024-09-14T08:31:36.695 INFO:teuthology.orchestra.run.smithi164.stdout:(12/12): python3-scipy-1.6.2-8.el9.x86_64.rpm 2.1 MB/s | 16 MB 00:07 2024-09-14T08:31:36.696 INFO:teuthology.orchestra.run.smithi164.stdout:-------------------------------------------------------------------------------- 2024-09-14T08:31:36.697 INFO:teuthology.orchestra.run.smithi164.stdout:Total 4.1 MB/s | 38 MB 00:09 2024-09-14T08:31:37.056 INFO:teuthology.orchestra.run.smithi164.stdout:Running transaction check 2024-09-14T08:31:37.083 INFO:teuthology.orchestra.run.smithi164.stdout:Transaction check succeeded. 2024-09-14T08:31:37.083 INFO:teuthology.orchestra.run.smithi164.stdout:Running transaction test 2024-09-14T08:31:37.509 INFO:teuthology.orchestra.run.smithi164.stdout:Transaction test succeeded. 2024-09-14T08:31:37.509 INFO:teuthology.orchestra.run.smithi164.stdout:Running transaction 2024-09-14T08:31:37.777 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:19.3.0-4775.g7f24e 12/12 2024-09-14T08:31:37.777 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : ceph-mgr-diskprediction-local-2:19.3.0-4775.g7f24e 1/12 2024-09-14T08:31:37.777 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : libgfortran-11.5.0-2.el9.x86_64 2/12 2024-09-14T08:31:37.778 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : libquadmath-11.5.0-2.el9.x86_64 3/12 2024-09-14T08:31:37.778 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : flexiblas-3.0.4-8.el9.x86_64 4/12 2024-09-14T08:31:37.778 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : flexiblas-netlib-3.0.4-8.el9.x86_64 5/12 2024-09-14T08:31:37.778 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 6/12 2024-09-14T08:31:37.778 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : openblas-0.3.26-2.el9.x86_64 7/12 2024-09-14T08:31:37.778 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : openblas-openmp-0.3.26-2.el9.x86_64 8/12 2024-09-14T08:31:37.778 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-devel-3.9.19-8.el9.x86_64 9/12 2024-09-14T08:31:37.778 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-numpy-1:1.20.1-5.el9.x86_64 10/12 2024-09-14T08:31:37.778 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 11/12 2024-09-14T08:31:38.207 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-scipy-1.6.2-8.el9.x86_64 12/12 2024-09-14T08:31:38.207 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:31:38.207 INFO:teuthology.orchestra.run.smithi135.stdout:Installed: 2024-09-14T08:31:38.207 INFO:teuthology.orchestra.run.smithi135.stdout: ceph-mgr-diskprediction-local-2:19.3.0-4775.g7f24e589.el9.noarch 2024-09-14T08:31:38.207 INFO:teuthology.orchestra.run.smithi135.stdout: flexiblas-3.0.4-8.el9.x86_64 2024-09-14T08:31:38.207 INFO:teuthology.orchestra.run.smithi135.stdout: flexiblas-netlib-3.0.4-8.el9.x86_64 2024-09-14T08:31:38.208 INFO:teuthology.orchestra.run.smithi135.stdout: flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 2024-09-14T08:31:38.208 INFO:teuthology.orchestra.run.smithi135.stdout: libgfortran-11.5.0-2.el9.x86_64 2024-09-14T08:31:38.208 INFO:teuthology.orchestra.run.smithi135.stdout: libquadmath-11.5.0-2.el9.x86_64 2024-09-14T08:31:38.208 INFO:teuthology.orchestra.run.smithi135.stdout: openblas-0.3.26-2.el9.x86_64 2024-09-14T08:31:38.208 INFO:teuthology.orchestra.run.smithi135.stdout: openblas-openmp-0.3.26-2.el9.x86_64 2024-09-14T08:31:38.208 INFO:teuthology.orchestra.run.smithi135.stdout: python3-devel-3.9.19-8.el9.x86_64 2024-09-14T08:31:38.208 INFO:teuthology.orchestra.run.smithi135.stdout: python3-numpy-1:1.20.1-5.el9.x86_64 2024-09-14T08:31:38.208 INFO:teuthology.orchestra.run.smithi135.stdout: python3-numpy-f2py-1:1.20.1-5.el9.x86_64 2024-09-14T08:31:38.208 INFO:teuthology.orchestra.run.smithi135.stdout: python3-scipy-1.6.2-8.el9.x86_64 2024-09-14T08:31:38.208 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:31:38.208 INFO:teuthology.orchestra.run.smithi135.stdout:Complete! 2024-09-14T08:31:38.281 INFO:teuthology.orchestra.run.smithi164.stdout: Preparing : 1/1 2024-09-14T08:31:38.368 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : flexiblas-3.0.4-8.el9.x86_64 1/12 2024-09-14T08:31:38.410 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : libquadmath-11.5.0-2.el9.x86_64 2/12 2024-09-14T08:31:38.513 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : libgfortran-11.5.0-2.el9.x86_64 3/12 2024-09-14T08:31:38.553 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : python3-devel-3.9.19-8.el9.x86_64 4/12 2024-09-14T08:31:38.560 DEBUG:teuthology.orchestra.run.smithi135:> sudo yum -y install ceph-mgr-rook 2024-09-14T08:31:38.739 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : openblas-0.3.26-2.el9.x86_64 5/12 2024-09-14T08:31:38.777 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : openblas-openmp-0.3.26-2.el9.x86_64 6/12 2024-09-14T08:31:38.853 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 7/12 2024-09-14T08:31:39.151 INFO:teuthology.orchestra.run.smithi135.stdout:Last metadata expiration check: 0:01:22 ago on Sat 14 Sep 2024 08:30:17 AM UTC. 2024-09-14T08:31:39.298 INFO:teuthology.orchestra.run.smithi135.stdout:Dependencies resolved. 2024-09-14T08:31:39.299 INFO:teuthology.orchestra.run.smithi135.stdout:================================================================================ 2024-09-14T08:31:39.300 INFO:teuthology.orchestra.run.smithi135.stdout: Package Arch Version Repository Size 2024-09-14T08:31:39.300 INFO:teuthology.orchestra.run.smithi135.stdout:================================================================================ 2024-09-14T08:31:39.300 INFO:teuthology.orchestra.run.smithi135.stdout:Installing: 2024-09-14T08:31:39.300 INFO:teuthology.orchestra.run.smithi135.stdout: ceph-mgr-rook noarch 2:19.3.0-4775.g7f24e589.el9 ceph-noarch 49 k 2024-09-14T08:31:39.300 INFO:teuthology.orchestra.run.smithi135.stdout:Installing dependencies: 2024-09-14T08:31:39.300 INFO:teuthology.orchestra.run.smithi135.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2024-09-14T08:31:39.300 INFO:teuthology.orchestra.run.smithi135.stdout: python3-certifi noarch 2023.05.07-4.el9 epel 14 k 2024-09-14T08:31:39.300 INFO:teuthology.orchestra.run.smithi135.stdout: python3-google-auth noarch 1:2.34.0-1.el9 epel 220 k 2024-09-14T08:31:39.300 INFO:teuthology.orchestra.run.smithi135.stdout: python3-jsonpatch noarch 1.21-16.el9 appstream 26 k 2024-09-14T08:31:39.300 INFO:teuthology.orchestra.run.smithi135.stdout: python3-jsonpointer noarch 2.0-4.el9 appstream 19 k 2024-09-14T08:31:39.301 INFO:teuthology.orchestra.run.smithi135.stdout: python3-kubernetes noarch 1:26.1.0-2.el9 epel 1.0 M 2024-09-14T08:31:39.301 INFO:teuthology.orchestra.run.smithi135.stdout: python3-oauthlib noarch 3.1.1-5.el9 appstream 222 k 2024-09-14T08:31:39.301 INFO:teuthology.orchestra.run.smithi135.stdout: python3-pyasn1 noarch 0.4.8-6.el9 appstream 159 k 2024-09-14T08:31:39.301 INFO:teuthology.orchestra.run.smithi135.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 appstream 279 k 2024-09-14T08:31:39.301 INFO:teuthology.orchestra.run.smithi135.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 appstream 54 k 2024-09-14T08:31:39.301 INFO:teuthology.orchestra.run.smithi135.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2024-09-14T08:31:39.301 INFO:teuthology.orchestra.run.smithi135.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2024-09-14T08:31:39.302 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:31:39.302 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction Summary 2024-09-14T08:31:39.302 INFO:teuthology.orchestra.run.smithi135.stdout:================================================================================ 2024-09-14T08:31:39.302 INFO:teuthology.orchestra.run.smithi135.stdout:Install 13 Packages 2024-09-14T08:31:39.302 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:31:39.303 INFO:teuthology.orchestra.run.smithi135.stdout:Total download size: 2.2 M 2024-09-14T08:31:39.303 INFO:teuthology.orchestra.run.smithi135.stdout:Installed size: 26 M 2024-09-14T08:31:39.304 INFO:teuthology.orchestra.run.smithi135.stdout:Downloading Packages: 2024-09-14T08:31:39.511 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : flexiblas-netlib-3.0.4-8.el9.x86_64 8/12 2024-09-14T08:31:39.624 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : python3-numpy-1:1.20.1-5.el9.x86_64 9/12 2024-09-14T08:31:39.846 INFO:teuthology.orchestra.run.smithi135.stdout:(1/13): python3-jsonpointer-2.0-4.el9.noarch.rp 174 kB/s | 19 kB 00:00 2024-09-14T08:31:39.871 INFO:teuthology.orchestra.run.smithi135.stdout:(2/13): python3-jsonpatch-1.21-16.el9.noarch.rp 190 kB/s | 26 kB 00:00 2024-09-14T08:31:39.896 INFO:teuthology.orchestra.run.smithi135.stdout:(3/13): ceph-mgr-rook-19.3.0-4775.g7f24e589.el9 307 kB/s | 49 kB 00:00 2024-09-14T08:31:39.988 INFO:teuthology.orchestra.run.smithi135.stdout:(4/13): python3-pyasn1-0.4.8-6.el9.noarch.rpm 1.3 MB/s | 159 kB 00:00 2024-09-14T08:31:40.013 INFO:teuthology.orchestra.run.smithi135.stdout:(5/13): python3-oauthlib-3.1.1-5.el9.noarch.rpm 1.3 MB/s | 222 kB 00:00 2024-09-14T08:31:40.039 INFO:teuthology.orchestra.run.smithi135.stdout:(6/13): python3-requests-oauthlib-1.3.0-12.el9. 1.0 MB/s | 54 kB 00:00 2024-09-14T08:31:40.081 INFO:teuthology.orchestra.run.smithi135.stdout:(7/13): python3-cachetools-4.2.4-1.el9.noarch.r 481 kB/s | 32 kB 00:00 2024-09-14T08:31:40.106 INFO:teuthology.orchestra.run.smithi135.stdout:(8/13): python3-pyasn1-modules-0.4.8-6.el9.noar 1.3 MB/s | 279 kB 00:00 2024-09-14T08:31:40.131 INFO:teuthology.orchestra.run.smithi135.stdout:(9/13): python3-certifi-2023.05.07-4.el9.noarch 153 kB/s | 14 kB 00:00 2024-09-14T08:31:40.165 INFO:teuthology.orchestra.run.smithi135.stdout:(10/13): python3-rsa-4.9-2.el9.noarch.rpm 2.2 MB/s | 59 kB 00:00 2024-09-14T08:31:40.190 INFO:teuthology.orchestra.run.smithi135.stdout:(11/13): python3-google-auth-2.34.0-1.el9.noarc 2.0 MB/s | 220 kB 00:00 2024-09-14T08:31:40.215 INFO:teuthology.orchestra.run.smithi135.stdout:(12/13): python3-websocket-client-1.2.3-2.el9.n 1.8 MB/s | 90 kB 00:00 2024-09-14T08:31:40.257 INFO:teuthology.orchestra.run.smithi135.stdout:(13/13): python3-kubernetes-26.1.0-2.el9.noarch 6.8 MB/s | 1.0 MB 00:00 2024-09-14T08:31:40.260 INFO:teuthology.orchestra.run.smithi135.stdout:-------------------------------------------------------------------------------- 2024-09-14T08:31:40.260 INFO:teuthology.orchestra.run.smithi135.stdout:Total 2.3 MB/s | 2.2 MB 00:00 2024-09-14T08:31:40.326 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction check 2024-09-14T08:31:40.342 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction check succeeded. 2024-09-14T08:31:40.342 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction test 2024-09-14T08:31:40.434 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction test succeeded. 2024-09-14T08:31:40.435 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction 2024-09-14T08:31:40.871 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 10/12 2024-09-14T08:31:40.942 INFO:teuthology.orchestra.run.smithi135.stdout: Preparing : 1/1 2024-09-14T08:31:41.122 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 1/13 2024-09-14T08:31:41.183 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 2/13 2024-09-14T08:31:41.247 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-rsa-4.9-2.el9.noarch 3/13 2024-09-14T08:31:41.284 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 4/13 2024-09-14T08:31:41.326 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-certifi-2023.05.07-4.el9.noarch 5/13 2024-09-14T08:31:41.436 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 6/13 2024-09-14T08:31:41.537 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-google-auth-1:2.34.0-1.el9.noarch 7/13 2024-09-14T08:31:41.594 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-oauthlib-3.1.1-5.el9.noarch 8/13 2024-09-14T08:31:41.907 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : python3-scipy-1.6.2-8.el9.x86_64 11/12 2024-09-14T08:31:41.935 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : ceph-mgr-diskprediction-local-2:19.3.0-4775.g7f24e 12/12 2024-09-14T08:31:42.309 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 9/13 2024-09-14T08:31:42.358 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-kubernetes-1:26.1.0-2.el9.noarch 10/13 2024-09-14T08:31:42.400 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-jsonpointer-2.0-4.el9.noarch 11/13 2024-09-14T08:31:42.457 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-jsonpatch-1.21-16.el9.noarch 12/13 2024-09-14T08:31:42.478 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : ceph-mgr-rook-2:19.3.0-4775.g7f24e589.el9.noarch 13/13 2024-09-14T08:31:43.261 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: ceph-mgr-rook-2:19.3.0-4775.g7f24e589.el9.noarch 13/13 2024-09-14T08:31:43.261 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : ceph-mgr-rook-2:19.3.0-4775.g7f24e589.el9.noarch 1/13 2024-09-14T08:31:43.261 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-jsonpatch-1.21-16.el9.noarch 2/13 2024-09-14T08:31:43.261 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-jsonpointer-2.0-4.el9.noarch 3/13 2024-09-14T08:31:43.261 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-oauthlib-3.1.1-5.el9.noarch 4/13 2024-09-14T08:31:43.261 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 5/13 2024-09-14T08:31:43.261 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 6/13 2024-09-14T08:31:43.262 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 7/13 2024-09-14T08:31:43.262 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 8/13 2024-09-14T08:31:43.262 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-certifi-2023.05.07-4.el9.noarch 9/13 2024-09-14T08:31:43.262 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-google-auth-1:2.34.0-1.el9.noarch 10/13 2024-09-14T08:31:43.262 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-kubernetes-1:26.1.0-2.el9.noarch 11/13 2024-09-14T08:31:43.262 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 12/13 2024-09-14T08:31:43.602 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 13/13 2024-09-14T08:31:43.603 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:31:43.603 INFO:teuthology.orchestra.run.smithi135.stdout:Installed: 2024-09-14T08:31:43.603 INFO:teuthology.orchestra.run.smithi135.stdout: ceph-mgr-rook-2:19.3.0-4775.g7f24e589.el9.noarch 2024-09-14T08:31:43.603 INFO:teuthology.orchestra.run.smithi135.stdout: python3-cachetools-4.2.4-1.el9.noarch 2024-09-14T08:31:43.603 INFO:teuthology.orchestra.run.smithi135.stdout: python3-certifi-2023.05.07-4.el9.noarch 2024-09-14T08:31:43.603 INFO:teuthology.orchestra.run.smithi135.stdout: python3-google-auth-1:2.34.0-1.el9.noarch 2024-09-14T08:31:43.603 INFO:teuthology.orchestra.run.smithi135.stdout: python3-jsonpatch-1.21-16.el9.noarch 2024-09-14T08:31:43.603 INFO:teuthology.orchestra.run.smithi135.stdout: python3-jsonpointer-2.0-4.el9.noarch 2024-09-14T08:31:43.603 INFO:teuthology.orchestra.run.smithi135.stdout: python3-kubernetes-1:26.1.0-2.el9.noarch 2024-09-14T08:31:43.603 INFO:teuthology.orchestra.run.smithi135.stdout: python3-oauthlib-3.1.1-5.el9.noarch 2024-09-14T08:31:43.603 INFO:teuthology.orchestra.run.smithi135.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2024-09-14T08:31:43.604 INFO:teuthology.orchestra.run.smithi135.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2024-09-14T08:31:43.604 INFO:teuthology.orchestra.run.smithi135.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2024-09-14T08:31:43.604 INFO:teuthology.orchestra.run.smithi135.stdout: python3-rsa-4.9-2.el9.noarch 2024-09-14T08:31:43.604 INFO:teuthology.orchestra.run.smithi135.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2024-09-14T08:31:43.604 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:31:43.604 INFO:teuthology.orchestra.run.smithi135.stdout:Complete! 2024-09-14T08:31:43.867 DEBUG:teuthology.orchestra.run.smithi135:> sudo yum -y install ceph-mgr-cephadm 2024-09-14T08:31:44.443 INFO:teuthology.orchestra.run.smithi135.stdout:Last metadata expiration check: 0:01:27 ago on Sat 14 Sep 2024 08:30:17 AM UTC. 2024-09-14T08:31:44.580 INFO:teuthology.orchestra.run.smithi135.stdout:Dependencies resolved. 2024-09-14T08:31:44.581 INFO:teuthology.orchestra.run.smithi135.stdout:================================================================================ 2024-09-14T08:31:44.581 INFO:teuthology.orchestra.run.smithi135.stdout: Package Arch Version Repository Size 2024-09-14T08:31:44.581 INFO:teuthology.orchestra.run.smithi135.stdout:================================================================================ 2024-09-14T08:31:44.581 INFO:teuthology.orchestra.run.smithi135.stdout:Installing: 2024-09-14T08:31:44.581 INFO:teuthology.orchestra.run.smithi135.stdout: ceph-mgr-cephadm noarch 2:19.3.0-4775.g7f24e589.el9 ceph-noarch 154 k 2024-09-14T08:31:44.581 INFO:teuthology.orchestra.run.smithi135.stdout:Installing dependencies: 2024-09-14T08:31:44.582 INFO:teuthology.orchestra.run.smithi135.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2024-09-14T08:31:44.582 INFO:teuthology.orchestra.run.smithi135.stdout: python3-babel noarch 2.9.1-2.el9 appstream 6.0 M 2024-09-14T08:31:44.582 INFO:teuthology.orchestra.run.smithi135.stdout: python3-jinja2 noarch 2.11.3-6.el9 appstream 248 k 2024-09-14T08:31:44.582 INFO:teuthology.orchestra.run.smithi135.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2024-09-14T08:31:44.582 INFO:teuthology.orchestra.run.smithi135.stdout: python3-typing-extensions noarch 4.12.2-2.el9 epel 74 k 2024-09-14T08:31:44.582 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:31:44.582 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction Summary 2024-09-14T08:31:44.582 INFO:teuthology.orchestra.run.smithi135.stdout:================================================================================ 2024-09-14T08:31:44.582 INFO:teuthology.orchestra.run.smithi135.stdout:Install 6 Packages 2024-09-14T08:31:44.583 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:31:44.583 INFO:teuthology.orchestra.run.smithi135.stdout:Total download size: 7.0 M 2024-09-14T08:31:44.583 INFO:teuthology.orchestra.run.smithi135.stdout:Installed size: 33 M 2024-09-14T08:31:44.584 INFO:teuthology.orchestra.run.smithi135.stdout:Downloading Packages: 2024-09-14T08:31:44.618 INFO:teuthology.orchestra.run.smithi164.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:19.3.0-4775.g7f24e 12/12 2024-09-14T08:31:44.619 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : ceph-mgr-diskprediction-local-2:19.3.0-4775.g7f24e 1/12 2024-09-14T08:31:44.619 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : libgfortran-11.5.0-2.el9.x86_64 2/12 2024-09-14T08:31:44.619 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : libquadmath-11.5.0-2.el9.x86_64 3/12 2024-09-14T08:31:44.619 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : flexiblas-3.0.4-8.el9.x86_64 4/12 2024-09-14T08:31:44.619 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : flexiblas-netlib-3.0.4-8.el9.x86_64 5/12 2024-09-14T08:31:44.619 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 6/12 2024-09-14T08:31:44.619 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : openblas-0.3.26-2.el9.x86_64 7/12 2024-09-14T08:31:44.619 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : openblas-openmp-0.3.26-2.el9.x86_64 8/12 2024-09-14T08:31:44.620 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : python3-devel-3.9.19-8.el9.x86_64 9/12 2024-09-14T08:31:44.620 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : python3-numpy-1:1.20.1-5.el9.x86_64 10/12 2024-09-14T08:31:44.620 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 11/12 2024-09-14T08:31:44.893 INFO:teuthology.orchestra.run.smithi135.stdout:(1/6): python3-jinja2-2.11.3-6.el9.noarch.rpm 1.2 MB/s | 248 kB 00:00 2024-09-14T08:31:44.918 INFO:teuthology.orchestra.run.smithi135.stdout:(2/6): ceph-mgr-cephadm-19.3.0-4775.g7f24e589.e 661 kB/s | 154 kB 00:00 2024-09-14T08:31:44.968 INFO:teuthology.orchestra.run.smithi135.stdout:(3/6): python3-asyncssh-2.13.2-5.el9.noarch.rpm 7.1 MB/s | 548 kB 00:00 2024-09-14T08:31:44.994 INFO:teuthology.orchestra.run.smithi135.stdout:(4/6): python3-natsort-7.1.1-5.el9.noarch.rpm 767 kB/s | 58 kB 00:00 2024-09-14T08:31:45.019 INFO:teuthology.orchestra.run.smithi135.stdout:(5/6): python3-typing-extensions-4.12.2-2.el9.n 1.4 MB/s | 74 kB 00:00 2024-09-14T08:31:45.063 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : python3-scipy-1.6.2-8.el9.x86_64 12/12 2024-09-14T08:31:45.063 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:31:45.063 INFO:teuthology.orchestra.run.smithi164.stdout:Installed: 2024-09-14T08:31:45.063 INFO:teuthology.orchestra.run.smithi164.stdout: ceph-mgr-diskprediction-local-2:19.3.0-4775.g7f24e589.el9.noarch 2024-09-14T08:31:45.063 INFO:teuthology.orchestra.run.smithi164.stdout: flexiblas-3.0.4-8.el9.x86_64 2024-09-14T08:31:45.064 INFO:teuthology.orchestra.run.smithi164.stdout: flexiblas-netlib-3.0.4-8.el9.x86_64 2024-09-14T08:31:45.064 INFO:teuthology.orchestra.run.smithi164.stdout: flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 2024-09-14T08:31:45.064 INFO:teuthology.orchestra.run.smithi164.stdout: libgfortran-11.5.0-2.el9.x86_64 2024-09-14T08:31:45.064 INFO:teuthology.orchestra.run.smithi164.stdout: libquadmath-11.5.0-2.el9.x86_64 2024-09-14T08:31:45.064 INFO:teuthology.orchestra.run.smithi164.stdout: openblas-0.3.26-2.el9.x86_64 2024-09-14T08:31:45.064 INFO:teuthology.orchestra.run.smithi164.stdout: openblas-openmp-0.3.26-2.el9.x86_64 2024-09-14T08:31:45.064 INFO:teuthology.orchestra.run.smithi164.stdout: python3-devel-3.9.19-8.el9.x86_64 2024-09-14T08:31:45.064 INFO:teuthology.orchestra.run.smithi164.stdout: python3-numpy-1:1.20.1-5.el9.x86_64 2024-09-14T08:31:45.064 INFO:teuthology.orchestra.run.smithi164.stdout: python3-numpy-f2py-1:1.20.1-5.el9.x86_64 2024-09-14T08:31:45.064 INFO:teuthology.orchestra.run.smithi164.stdout: python3-scipy-1.6.2-8.el9.x86_64 2024-09-14T08:31:45.064 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:31:45.064 INFO:teuthology.orchestra.run.smithi164.stdout:Complete! 2024-09-14T08:31:45.508 DEBUG:teuthology.orchestra.run.smithi164:> sudo yum -y install ceph-mgr-rook 2024-09-14T08:31:46.088 INFO:teuthology.orchestra.run.smithi164.stdout:Last metadata expiration check: 0:01:30 ago on Sat 14 Sep 2024 08:30:16 AM UTC. 2024-09-14T08:31:46.225 INFO:teuthology.orchestra.run.smithi164.stdout:Dependencies resolved. 2024-09-14T08:31:46.226 INFO:teuthology.orchestra.run.smithi164.stdout:================================================================================ 2024-09-14T08:31:46.226 INFO:teuthology.orchestra.run.smithi164.stdout: Package Arch Version Repository Size 2024-09-14T08:31:46.226 INFO:teuthology.orchestra.run.smithi164.stdout:================================================================================ 2024-09-14T08:31:46.226 INFO:teuthology.orchestra.run.smithi164.stdout:Installing: 2024-09-14T08:31:46.227 INFO:teuthology.orchestra.run.smithi164.stdout: ceph-mgr-rook noarch 2:19.3.0-4775.g7f24e589.el9 ceph-noarch 49 k 2024-09-14T08:31:46.227 INFO:teuthology.orchestra.run.smithi164.stdout:Installing dependencies: 2024-09-14T08:31:46.227 INFO:teuthology.orchestra.run.smithi164.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2024-09-14T08:31:46.227 INFO:teuthology.orchestra.run.smithi164.stdout: python3-certifi noarch 2023.05.07-4.el9 epel 14 k 2024-09-14T08:31:46.227 INFO:teuthology.orchestra.run.smithi164.stdout: python3-google-auth noarch 1:2.34.0-1.el9 epel 220 k 2024-09-14T08:31:46.227 INFO:teuthology.orchestra.run.smithi164.stdout: python3-jsonpatch noarch 1.21-16.el9 appstream 26 k 2024-09-14T08:31:46.227 INFO:teuthology.orchestra.run.smithi164.stdout: python3-jsonpointer noarch 2.0-4.el9 appstream 19 k 2024-09-14T08:31:46.227 INFO:teuthology.orchestra.run.smithi164.stdout: python3-kubernetes noarch 1:26.1.0-2.el9 epel 1.0 M 2024-09-14T08:31:46.227 INFO:teuthology.orchestra.run.smithi164.stdout: python3-oauthlib noarch 3.1.1-5.el9 appstream 222 k 2024-09-14T08:31:46.227 INFO:teuthology.orchestra.run.smithi164.stdout: python3-pyasn1 noarch 0.4.8-6.el9 appstream 159 k 2024-09-14T08:31:46.227 INFO:teuthology.orchestra.run.smithi164.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 appstream 279 k 2024-09-14T08:31:46.228 INFO:teuthology.orchestra.run.smithi164.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 appstream 54 k 2024-09-14T08:31:46.228 INFO:teuthology.orchestra.run.smithi164.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2024-09-14T08:31:46.228 INFO:teuthology.orchestra.run.smithi164.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2024-09-14T08:31:46.228 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:31:46.228 INFO:teuthology.orchestra.run.smithi164.stdout:Transaction Summary 2024-09-14T08:31:46.228 INFO:teuthology.orchestra.run.smithi164.stdout:================================================================================ 2024-09-14T08:31:46.228 INFO:teuthology.orchestra.run.smithi164.stdout:Install 13 Packages 2024-09-14T08:31:46.228 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:31:46.229 INFO:teuthology.orchestra.run.smithi164.stdout:Total download size: 2.2 M 2024-09-14T08:31:46.229 INFO:teuthology.orchestra.run.smithi164.stdout:Installed size: 26 M 2024-09-14T08:31:46.229 INFO:teuthology.orchestra.run.smithi164.stdout:Downloading Packages: 2024-09-14T08:31:46.602 INFO:teuthology.orchestra.run.smithi135.stdout:(6/6): python3-babel-2.9.1-2.el9.noarch.rpm 3.1 MB/s | 6.0 MB 00:01 2024-09-14T08:31:46.605 INFO:teuthology.orchestra.run.smithi135.stdout:-------------------------------------------------------------------------------- 2024-09-14T08:31:46.605 INFO:teuthology.orchestra.run.smithi135.stdout:Total 3.5 MB/s | 7.0 MB 00:02 2024-09-14T08:31:46.611 INFO:teuthology.orchestra.run.smithi164.stdout:(1/13): python3-jsonpatch-1.21-16.el9.noarch.rp 342 kB/s | 26 kB 00:00 2024-09-14T08:31:46.637 INFO:teuthology.orchestra.run.smithi164.stdout:(2/13): python3-jsonpointer-2.0-4.el9.noarch.rp 191 kB/s | 19 kB 00:00 2024-09-14T08:31:46.679 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction check 2024-09-14T08:31:46.691 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction check succeeded. 2024-09-14T08:31:46.691 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction test 2024-09-14T08:31:46.712 INFO:teuthology.orchestra.run.smithi164.stdout:(3/13): python3-pyasn1-0.4.8-6.el9.noarch.rpm 2.1 MB/s | 159 kB 00:00 2024-09-14T08:31:46.737 INFO:teuthology.orchestra.run.smithi164.stdout:(4/13): python3-oauthlib-3.1.1-5.el9.noarch.rpm 2.2 MB/s | 222 kB 00:00 2024-09-14T08:31:46.763 INFO:teuthology.orchestra.run.smithi164.stdout:(5/13): ceph-mgr-rook-19.3.0-4775.g7f24e589.el9 218 kB/s | 49 kB 00:00 2024-09-14T08:31:46.793 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction test succeeded. 2024-09-14T08:31:46.796 INFO:teuthology.orchestra.run.smithi164.stdout:(6/13): python3-requests-oauthlib-1.3.0-12.el9. 911 kB/s | 54 kB 00:00 2024-09-14T08:31:46.798 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction 2024-09-14T08:31:46.830 INFO:teuthology.orchestra.run.smithi164.stdout:(7/13): python3-pyasn1-modules-0.4.8-6.el9.noar 2.3 MB/s | 279 kB 00:00 2024-09-14T08:31:46.872 INFO:teuthology.orchestra.run.smithi164.stdout:(8/13): python3-cachetools-4.2.4-1.el9.noarch.r 295 kB/s | 32 kB 00:00 2024-09-14T08:31:46.889 INFO:teuthology.orchestra.run.smithi164.stdout:(9/13): python3-certifi-2023.05.07-4.el9.noarch 153 kB/s | 14 kB 00:00 2024-09-14T08:31:46.922 INFO:teuthology.orchestra.run.smithi164.stdout:(10/13): python3-rsa-4.9-2.el9.noarch.rpm 1.7 MB/s | 59 kB 00:00 2024-09-14T08:31:46.948 INFO:teuthology.orchestra.run.smithi164.stdout:(11/13): python3-google-auth-2.34.0-1.el9.noarc 2.1 MB/s | 220 kB 00:00 2024-09-14T08:31:46.973 INFO:teuthology.orchestra.run.smithi164.stdout:(12/13): python3-websocket-client-1.2.3-2.el9.n 1.7 MB/s | 90 kB 00:00 2024-09-14T08:31:47.015 INFO:teuthology.orchestra.run.smithi164.stdout:(13/13): python3-kubernetes-26.1.0-2.el9.noarch 7.2 MB/s | 1.0 MB 00:00 2024-09-14T08:31:47.018 INFO:teuthology.orchestra.run.smithi164.stdout:-------------------------------------------------------------------------------- 2024-09-14T08:31:47.018 INFO:teuthology.orchestra.run.smithi164.stdout:Total 2.8 MB/s | 2.2 MB 00:00 2024-09-14T08:31:47.083 INFO:teuthology.orchestra.run.smithi164.stdout:Running transaction check 2024-09-14T08:31:47.098 INFO:teuthology.orchestra.run.smithi164.stdout:Transaction check succeeded. 2024-09-14T08:31:47.099 INFO:teuthology.orchestra.run.smithi164.stdout:Running transaction test 2024-09-14T08:31:47.103 INFO:teuthology.orchestra.run.smithi135.stdout: Preparing : 1/1 2024-09-14T08:31:47.189 INFO:teuthology.orchestra.run.smithi164.stdout:Transaction test succeeded. 2024-09-14T08:31:47.192 INFO:teuthology.orchestra.run.smithi164.stdout:Running transaction 2024-09-14T08:31:47.273 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-typing-extensions-4.12.2-2.el9.noarch 1/6 2024-09-14T08:31:47.328 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 2/6 2024-09-14T08:31:47.703 INFO:teuthology.orchestra.run.smithi164.stdout: Preparing : 1/1 2024-09-14T08:31:47.845 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 3/6 2024-09-14T08:31:47.872 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 1/13 2024-09-14T08:31:47.933 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 2/13 2024-09-14T08:31:47.939 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 4/6 2024-09-14T08:31:47.989 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : python3-rsa-4.9-2.el9.noarch 3/13 2024-09-14T08:31:48.007 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : python3-jinja2-2.11.3-6.el9.noarch 5/6 2024-09-14T08:31:48.025 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 4/13 2024-09-14T08:31:48.032 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : ceph-mgr-cephadm-2:19.3.0-4775.g7f24e589.el9.noarch 6/6 2024-09-14T08:31:48.067 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : python3-certifi-2023.05.07-4.el9.noarch 5/13 2024-09-14T08:31:48.170 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 6/13 2024-09-14T08:31:48.296 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : python3-google-auth-1:2.34.0-1.el9.noarch 7/13 2024-09-14T08:31:48.352 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : python3-oauthlib-3.1.1-5.el9.noarch 8/13 2024-09-14T08:31:48.806 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: ceph-mgr-cephadm-2:19.3.0-4775.g7f24e589.el9.noarch 6/6 2024-09-14T08:31:48.806 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : ceph-mgr-cephadm-2:19.3.0-4775.g7f24e589.el9.noarch 1/6 2024-09-14T08:31:48.806 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 2/6 2024-09-14T08:31:48.806 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-jinja2-2.11.3-6.el9.noarch 3/6 2024-09-14T08:31:48.806 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 4/6 2024-09-14T08:31:48.806 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 5/6 2024-09-14T08:31:49.070 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 9/13 2024-09-14T08:31:49.093 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : python3-typing-extensions-4.12.2-2.el9.noarch 6/6 2024-09-14T08:31:49.093 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:31:49.093 INFO:teuthology.orchestra.run.smithi135.stdout:Installed: 2024-09-14T08:31:49.093 INFO:teuthology.orchestra.run.smithi135.stdout: ceph-mgr-cephadm-2:19.3.0-4775.g7f24e589.el9.noarch 2024-09-14T08:31:49.093 INFO:teuthology.orchestra.run.smithi135.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2024-09-14T08:31:49.093 INFO:teuthology.orchestra.run.smithi135.stdout: python3-babel-2.9.1-2.el9.noarch 2024-09-14T08:31:49.093 INFO:teuthology.orchestra.run.smithi135.stdout: python3-jinja2-2.11.3-6.el9.noarch 2024-09-14T08:31:49.093 INFO:teuthology.orchestra.run.smithi135.stdout: python3-natsort-7.1.1-5.el9.noarch 2024-09-14T08:31:49.093 INFO:teuthology.orchestra.run.smithi135.stdout: python3-typing-extensions-4.12.2-2.el9.noarch 2024-09-14T08:31:49.093 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:31:49.093 INFO:teuthology.orchestra.run.smithi135.stdout:Complete! 2024-09-14T08:31:49.125 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : python3-kubernetes-1:26.1.0-2.el9.noarch 10/13 2024-09-14T08:31:49.158 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : python3-jsonpointer-2.0-4.el9.noarch 11/13 2024-09-14T08:31:49.206 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : python3-jsonpatch-1.21-16.el9.noarch 12/13 2024-09-14T08:31:49.228 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : ceph-mgr-rook-2:19.3.0-4775.g7f24e589.el9.noarch 13/13 2024-09-14T08:31:49.362 DEBUG:teuthology.orchestra.run.smithi135:> sudo yum -y install ceph-fuse 2024-09-14T08:31:49.930 INFO:teuthology.orchestra.run.smithi135.stdout:Last metadata expiration check: 0:01:32 ago on Sat 14 Sep 2024 08:30:17 AM UTC. 2024-09-14T08:31:50.066 INFO:teuthology.orchestra.run.smithi135.stdout:Dependencies resolved. 2024-09-14T08:31:50.066 INFO:teuthology.orchestra.run.smithi135.stdout:================================================================================ 2024-09-14T08:31:50.067 INFO:teuthology.orchestra.run.smithi135.stdout: Package Arch Version Repository Size 2024-09-14T08:31:50.067 INFO:teuthology.orchestra.run.smithi135.stdout:================================================================================ 2024-09-14T08:31:50.067 INFO:teuthology.orchestra.run.smithi135.stdout:Installing: 2024-09-14T08:31:50.067 INFO:teuthology.orchestra.run.smithi135.stdout: ceph-fuse x86_64 2:19.3.0-4775.g7f24e589.el9 ceph 900 k 2024-09-14T08:31:50.067 INFO:teuthology.orchestra.run.smithi135.stdout:Installing dependencies: 2024-09-14T08:31:50.067 INFO:teuthology.orchestra.run.smithi135.stdout: fuse x86_64 2.9.9-16.el9 baseos 80 k 2024-09-14T08:31:50.067 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:31:50.067 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction Summary 2024-09-14T08:31:50.067 INFO:teuthology.orchestra.run.smithi135.stdout:================================================================================ 2024-09-14T08:31:50.067 INFO:teuthology.orchestra.run.smithi135.stdout:Install 2 Packages 2024-09-14T08:31:50.067 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:31:50.068 INFO:teuthology.orchestra.run.smithi135.stdout:Total download size: 979 k 2024-09-14T08:31:50.068 INFO:teuthology.orchestra.run.smithi135.stdout:Installed size: 2.9 M 2024-09-14T08:31:50.068 INFO:teuthology.orchestra.run.smithi135.stdout:Downloading Packages: 2024-09-14T08:31:50.069 INFO:teuthology.orchestra.run.smithi164.stdout: Running scriptlet: ceph-mgr-rook-2:19.3.0-4775.g7f24e589.el9.noarch 13/13 2024-09-14T08:31:50.070 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : ceph-mgr-rook-2:19.3.0-4775.g7f24e589.el9.noarch 1/13 2024-09-14T08:31:50.070 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : python3-jsonpatch-1.21-16.el9.noarch 2/13 2024-09-14T08:31:50.070 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : python3-jsonpointer-2.0-4.el9.noarch 3/13 2024-09-14T08:31:50.070 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : python3-oauthlib-3.1.1-5.el9.noarch 4/13 2024-09-14T08:31:50.070 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 5/13 2024-09-14T08:31:50.070 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 6/13 2024-09-14T08:31:50.070 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 7/13 2024-09-14T08:31:50.070 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 8/13 2024-09-14T08:31:50.070 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : python3-certifi-2023.05.07-4.el9.noarch 9/13 2024-09-14T08:31:50.070 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : python3-google-auth-1:2.34.0-1.el9.noarch 10/13 2024-09-14T08:31:50.070 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : python3-kubernetes-1:26.1.0-2.el9.noarch 11/13 2024-09-14T08:31:50.070 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 12/13 2024-09-14T08:31:50.410 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 13/13 2024-09-14T08:31:50.410 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:31:50.410 INFO:teuthology.orchestra.run.smithi164.stdout:Installed: 2024-09-14T08:31:50.411 INFO:teuthology.orchestra.run.smithi164.stdout: ceph-mgr-rook-2:19.3.0-4775.g7f24e589.el9.noarch 2024-09-14T08:31:50.411 INFO:teuthology.orchestra.run.smithi164.stdout: python3-cachetools-4.2.4-1.el9.noarch 2024-09-14T08:31:50.411 INFO:teuthology.orchestra.run.smithi164.stdout: python3-certifi-2023.05.07-4.el9.noarch 2024-09-14T08:31:50.411 INFO:teuthology.orchestra.run.smithi164.stdout: python3-google-auth-1:2.34.0-1.el9.noarch 2024-09-14T08:31:50.411 INFO:teuthology.orchestra.run.smithi164.stdout: python3-jsonpatch-1.21-16.el9.noarch 2024-09-14T08:31:50.411 INFO:teuthology.orchestra.run.smithi164.stdout: python3-jsonpointer-2.0-4.el9.noarch 2024-09-14T08:31:50.411 INFO:teuthology.orchestra.run.smithi164.stdout: python3-kubernetes-1:26.1.0-2.el9.noarch 2024-09-14T08:31:50.411 INFO:teuthology.orchestra.run.smithi164.stdout: python3-oauthlib-3.1.1-5.el9.noarch 2024-09-14T08:31:50.411 INFO:teuthology.orchestra.run.smithi164.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2024-09-14T08:31:50.411 INFO:teuthology.orchestra.run.smithi164.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2024-09-14T08:31:50.411 INFO:teuthology.orchestra.run.smithi164.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2024-09-14T08:31:50.411 INFO:teuthology.orchestra.run.smithi164.stdout: python3-rsa-4.9-2.el9.noarch 2024-09-14T08:31:50.411 INFO:teuthology.orchestra.run.smithi164.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2024-09-14T08:31:50.411 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:31:50.412 INFO:teuthology.orchestra.run.smithi164.stdout:Complete! 2024-09-14T08:31:50.454 INFO:teuthology.orchestra.run.smithi135.stdout:(1/2): fuse-2.9.9-16.el9.x86_64.rpm 731 kB/s | 80 kB 00:00 2024-09-14T08:31:50.663 INFO:teuthology.orchestra.run.smithi135.stdout:(2/2): ceph-fuse-19.3.0-4775.g7f24e589.el9.x86_ 2.8 MB/s | 900 kB 00:00 2024-09-14T08:31:50.663 INFO:teuthology.orchestra.run.smithi135.stdout:-------------------------------------------------------------------------------- 2024-09-14T08:31:50.664 INFO:teuthology.orchestra.run.smithi135.stdout:Total 1.6 MB/s | 979 kB 00:00 2024-09-14T08:31:50.667 DEBUG:teuthology.orchestra.run.smithi164:> sudo yum -y install ceph-mgr-cephadm 2024-09-14T08:31:50.675 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction check 2024-09-14T08:31:50.698 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction check succeeded. 2024-09-14T08:31:50.698 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction test 2024-09-14T08:31:50.778 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction test succeeded. 2024-09-14T08:31:50.778 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction 2024-09-14T08:31:50.963 INFO:teuthology.orchestra.run.smithi135.stdout: Preparing : 1/1 2024-09-14T08:31:51.107 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : fuse-2.9.9-16.el9.x86_64 1/2 2024-09-14T08:31:51.186 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : ceph-fuse-2:19.3.0-4775.g7f24e589.el9.x86_64 2/2 2024-09-14T08:31:51.239 INFO:teuthology.orchestra.run.smithi164.stdout:Last metadata expiration check: 0:01:35 ago on Sat 14 Sep 2024 08:30:16 AM UTC. 2024-09-14T08:31:51.373 INFO:teuthology.orchestra.run.smithi164.stdout:Dependencies resolved. 2024-09-14T08:31:51.374 INFO:teuthology.orchestra.run.smithi164.stdout:================================================================================ 2024-09-14T08:31:51.374 INFO:teuthology.orchestra.run.smithi164.stdout: Package Arch Version Repository Size 2024-09-14T08:31:51.374 INFO:teuthology.orchestra.run.smithi164.stdout:================================================================================ 2024-09-14T08:31:51.374 INFO:teuthology.orchestra.run.smithi164.stdout:Installing: 2024-09-14T08:31:51.374 INFO:teuthology.orchestra.run.smithi164.stdout: ceph-mgr-cephadm noarch 2:19.3.0-4775.g7f24e589.el9 ceph-noarch 154 k 2024-09-14T08:31:51.374 INFO:teuthology.orchestra.run.smithi164.stdout:Installing dependencies: 2024-09-14T08:31:51.374 INFO:teuthology.orchestra.run.smithi164.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2024-09-14T08:31:51.374 INFO:teuthology.orchestra.run.smithi164.stdout: python3-babel noarch 2.9.1-2.el9 appstream 6.0 M 2024-09-14T08:31:51.374 INFO:teuthology.orchestra.run.smithi164.stdout: python3-jinja2 noarch 2.11.3-6.el9 appstream 248 k 2024-09-14T08:31:51.374 INFO:teuthology.orchestra.run.smithi164.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2024-09-14T08:31:51.375 INFO:teuthology.orchestra.run.smithi164.stdout: python3-typing-extensions noarch 4.12.2-2.el9 epel 74 k 2024-09-14T08:31:51.375 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:31:51.375 INFO:teuthology.orchestra.run.smithi164.stdout:Transaction Summary 2024-09-14T08:31:51.375 INFO:teuthology.orchestra.run.smithi164.stdout:================================================================================ 2024-09-14T08:31:51.375 INFO:teuthology.orchestra.run.smithi164.stdout:Install 6 Packages 2024-09-14T08:31:51.375 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:31:51.375 INFO:teuthology.orchestra.run.smithi164.stdout:Total download size: 7.0 M 2024-09-14T08:31:51.375 INFO:teuthology.orchestra.run.smithi164.stdout:Installed size: 33 M 2024-09-14T08:31:51.376 INFO:teuthology.orchestra.run.smithi164.stdout:Downloading Packages: 2024-09-14T08:31:51.743 INFO:teuthology.orchestra.run.smithi164.stdout:(1/6): python3-jinja2-2.11.3-6.el9.noarch.rpm 1.4 MB/s | 248 kB 00:00 2024-09-14T08:31:51.776 INFO:teuthology.orchestra.run.smithi164.stdout:(2/6): ceph-mgr-cephadm-19.3.0-4775.g7f24e589.e 722 kB/s | 154 kB 00:00 2024-09-14T08:31:51.826 INFO:teuthology.orchestra.run.smithi164.stdout:(3/6): python3-natsort-7.1.1-5.el9.noarch.rpm 1.1 MB/s | 58 kB 00:00 2024-09-14T08:31:51.860 INFO:teuthology.orchestra.run.smithi164.stdout:(4/6): python3-asyncssh-2.13.2-5.el9.noarch.rpm 4.6 MB/s | 548 kB 00:00 2024-09-14T08:31:51.864 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: ceph-fuse-2:19.3.0-4775.g7f24e589.el9.x86_64 2/2 2024-09-14T08:31:51.865 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : ceph-fuse-2:19.3.0-4775.g7f24e589.el9.x86_64 1/2 2024-09-14T08:31:51.885 INFO:teuthology.orchestra.run.smithi164.stdout:(5/6): python3-typing-extensions-4.12.2-2.el9.n 1.2 MB/s | 74 kB 00:00 2024-09-14T08:31:51.969 INFO:teuthology.orchestra.run.smithi164.stdout:(6/6): python3-babel-2.9.1-2.el9.noarch.rpm 15 MB/s | 6.0 MB 00:00 2024-09-14T08:31:51.971 INFO:teuthology.orchestra.run.smithi164.stdout:-------------------------------------------------------------------------------- 2024-09-14T08:31:51.971 INFO:teuthology.orchestra.run.smithi164.stdout:Total 12 MB/s | 7.0 MB 00:00 2024-09-14T08:31:52.042 INFO:teuthology.orchestra.run.smithi164.stdout:Running transaction check 2024-09-14T08:31:52.053 INFO:teuthology.orchestra.run.smithi164.stdout:Transaction check succeeded. 2024-09-14T08:31:52.053 INFO:teuthology.orchestra.run.smithi164.stdout:Running transaction test 2024-09-14T08:31:52.152 INFO:teuthology.orchestra.run.smithi164.stdout:Transaction test succeeded. 2024-09-14T08:31:52.152 INFO:teuthology.orchestra.run.smithi164.stdout:Running transaction 2024-09-14T08:31:52.159 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : fuse-2.9.9-16.el9.x86_64 2/2 2024-09-14T08:31:52.159 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:31:52.159 INFO:teuthology.orchestra.run.smithi135.stdout:Installed: 2024-09-14T08:31:52.159 INFO:teuthology.orchestra.run.smithi135.stdout: ceph-fuse-2:19.3.0-4775.g7f24e589.el9.x86_64 fuse-2.9.9-16.el9.x86_64 2024-09-14T08:31:52.159 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:31:52.159 INFO:teuthology.orchestra.run.smithi135.stdout:Complete! 2024-09-14T08:31:52.469 DEBUG:teuthology.orchestra.run.smithi135:> sudo yum -y install ceph-volume 2024-09-14T08:31:52.488 INFO:teuthology.orchestra.run.smithi164.stdout: Preparing : 1/1 2024-09-14T08:31:52.658 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : python3-typing-extensions-4.12.2-2.el9.noarch 1/6 2024-09-14T08:31:52.704 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 2/6 2024-09-14T08:31:53.053 INFO:teuthology.orchestra.run.smithi135.stdout:Last metadata expiration check: 0:01:36 ago on Sat 14 Sep 2024 08:30:17 AM UTC. 2024-09-14T08:31:53.192 INFO:teuthology.orchestra.run.smithi135.stdout:Dependencies resolved. 2024-09-14T08:31:53.193 INFO:teuthology.orchestra.run.smithi135.stdout:================================================================================ 2024-09-14T08:31:53.193 INFO:teuthology.orchestra.run.smithi135.stdout: Package Arch Version Repository Size 2024-09-14T08:31:53.193 INFO:teuthology.orchestra.run.smithi135.stdout:================================================================================ 2024-09-14T08:31:53.193 INFO:teuthology.orchestra.run.smithi135.stdout:Installing: 2024-09-14T08:31:53.193 INFO:teuthology.orchestra.run.smithi135.stdout: ceph-volume noarch 2:19.3.0-4775.g7f24e589.el9 ceph-noarch 285 k 2024-09-14T08:31:53.194 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:31:53.194 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction Summary 2024-09-14T08:31:53.194 INFO:teuthology.orchestra.run.smithi135.stdout:================================================================================ 2024-09-14T08:31:53.194 INFO:teuthology.orchestra.run.smithi135.stdout:Install 1 Package 2024-09-14T08:31:53.194 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:31:53.194 INFO:teuthology.orchestra.run.smithi135.stdout:Total download size: 285 k 2024-09-14T08:31:53.194 INFO:teuthology.orchestra.run.smithi135.stdout:Installed size: 1.3 M 2024-09-14T08:31:53.194 INFO:teuthology.orchestra.run.smithi135.stdout:Downloading Packages: 2024-09-14T08:31:53.224 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 3/6 2024-09-14T08:31:53.308 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 4/6 2024-09-14T08:31:53.375 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : python3-jinja2-2.11.3-6.el9.noarch 5/6 2024-09-14T08:31:53.400 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : ceph-mgr-cephadm-2:19.3.0-4775.g7f24e589.el9.noarch 6/6 2024-09-14T08:31:53.408 INFO:teuthology.orchestra.run.smithi135.stdout:ceph-volume-19.3.0-4775.g7f24e589.el9.noarch.rp 1.3 MB/s | 285 kB 00:00 2024-09-14T08:31:53.409 INFO:teuthology.orchestra.run.smithi135.stdout:-------------------------------------------------------------------------------- 2024-09-14T08:31:53.409 INFO:teuthology.orchestra.run.smithi135.stdout:Total 1.3 MB/s | 285 kB 00:00 2024-09-14T08:31:53.409 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction check 2024-09-14T08:31:53.417 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction check succeeded. 2024-09-14T08:31:53.418 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction test 2024-09-14T08:31:53.446 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction test succeeded. 2024-09-14T08:31:53.446 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction 2024-09-14T08:31:53.667 INFO:teuthology.orchestra.run.smithi135.stdout: Preparing : 1/1 2024-09-14T08:31:53.727 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : ceph-volume-2:19.3.0-4775.g7f24e589.el9.noarch 1/1 2024-09-14T08:31:53.744 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: ceph-volume-2:19.3.0-4775.g7f24e589.el9.noarch 1/1 2024-09-14T08:31:53.744 INFO:teuthology.orchestra.run.smithi135.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-14T08:31:53.744 INFO:teuthology.orchestra.run.smithi135.stdout:Invalid unit name "ceph-volume@*.service" escaped as "ceph-volume@\x2a.service". 2024-09-14T08:31:53.744 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:31:54.194 INFO:teuthology.orchestra.run.smithi164.stdout: Running scriptlet: ceph-mgr-cephadm-2:19.3.0-4775.g7f24e589.el9.noarch 6/6 2024-09-14T08:31:54.194 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : ceph-mgr-cephadm-2:19.3.0-4775.g7f24e589.el9.noarch 1/6 2024-09-14T08:31:54.194 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 2/6 2024-09-14T08:31:54.194 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : python3-jinja2-2.11.3-6.el9.noarch 3/6 2024-09-14T08:31:54.194 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 4/6 2024-09-14T08:31:54.195 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 5/6 2024-09-14T08:31:54.501 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : python3-typing-extensions-4.12.2-2.el9.noarch 6/6 2024-09-14T08:31:54.501 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:31:54.501 INFO:teuthology.orchestra.run.smithi164.stdout:Installed: 2024-09-14T08:31:54.501 INFO:teuthology.orchestra.run.smithi164.stdout: ceph-mgr-cephadm-2:19.3.0-4775.g7f24e589.el9.noarch 2024-09-14T08:31:54.501 INFO:teuthology.orchestra.run.smithi164.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2024-09-14T08:31:54.501 INFO:teuthology.orchestra.run.smithi164.stdout: python3-babel-2.9.1-2.el9.noarch 2024-09-14T08:31:54.501 INFO:teuthology.orchestra.run.smithi164.stdout: python3-jinja2-2.11.3-6.el9.noarch 2024-09-14T08:31:54.501 INFO:teuthology.orchestra.run.smithi164.stdout: python3-natsort-7.1.1-5.el9.noarch 2024-09-14T08:31:54.501 INFO:teuthology.orchestra.run.smithi164.stdout: python3-typing-extensions-4.12.2-2.el9.noarch 2024-09-14T08:31:54.501 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:31:54.502 INFO:teuthology.orchestra.run.smithi164.stdout:Complete! 2024-09-14T08:31:54.767 DEBUG:teuthology.orchestra.run.smithi164:> sudo yum -y install ceph-fuse 2024-09-14T08:31:54.768 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : ceph-volume-2:19.3.0-4775.g7f24e589.el9.noarch 1/1 2024-09-14T08:31:54.768 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:31:54.769 INFO:teuthology.orchestra.run.smithi135.stdout:Installed: 2024-09-14T08:31:54.769 INFO:teuthology.orchestra.run.smithi135.stdout: ceph-volume-2:19.3.0-4775.g7f24e589.el9.noarch 2024-09-14T08:31:54.769 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:31:54.769 INFO:teuthology.orchestra.run.smithi135.stdout:Complete! 2024-09-14T08:31:55.097 DEBUG:teuthology.orchestra.run.smithi135:> sudo yum -y install librados-devel 2024-09-14T08:31:55.334 INFO:teuthology.orchestra.run.smithi164.stdout:Last metadata expiration check: 0:01:39 ago on Sat 14 Sep 2024 08:30:16 AM UTC. 2024-09-14T08:31:55.467 INFO:teuthology.orchestra.run.smithi164.stdout:Dependencies resolved. 2024-09-14T08:31:55.468 INFO:teuthology.orchestra.run.smithi164.stdout:================================================================================ 2024-09-14T08:31:55.468 INFO:teuthology.orchestra.run.smithi164.stdout: Package Arch Version Repository Size 2024-09-14T08:31:55.468 INFO:teuthology.orchestra.run.smithi164.stdout:================================================================================ 2024-09-14T08:31:55.468 INFO:teuthology.orchestra.run.smithi164.stdout:Installing: 2024-09-14T08:31:55.468 INFO:teuthology.orchestra.run.smithi164.stdout: ceph-fuse x86_64 2:19.3.0-4775.g7f24e589.el9 ceph 900 k 2024-09-14T08:31:55.468 INFO:teuthology.orchestra.run.smithi164.stdout:Installing dependencies: 2024-09-14T08:31:55.468 INFO:teuthology.orchestra.run.smithi164.stdout: fuse x86_64 2.9.9-16.el9 baseos 80 k 2024-09-14T08:31:55.468 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:31:55.468 INFO:teuthology.orchestra.run.smithi164.stdout:Transaction Summary 2024-09-14T08:31:55.468 INFO:teuthology.orchestra.run.smithi164.stdout:================================================================================ 2024-09-14T08:31:55.469 INFO:teuthology.orchestra.run.smithi164.stdout:Install 2 Packages 2024-09-14T08:31:55.469 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:31:55.469 INFO:teuthology.orchestra.run.smithi164.stdout:Total download size: 979 k 2024-09-14T08:31:55.469 INFO:teuthology.orchestra.run.smithi164.stdout:Installed size: 2.9 M 2024-09-14T08:31:55.469 INFO:teuthology.orchestra.run.smithi164.stdout:Downloading Packages: 2024-09-14T08:31:55.638 INFO:teuthology.orchestra.run.smithi164.stdout:(1/2): fuse-2.9.9-16.el9.x86_64.rpm 651 kB/s | 80 kB 00:00 2024-09-14T08:31:55.689 INFO:teuthology.orchestra.run.smithi135.stdout:Last metadata expiration check: 0:01:38 ago on Sat 14 Sep 2024 08:30:17 AM UTC. 2024-09-14T08:31:55.825 INFO:teuthology.orchestra.run.smithi135.stdout:Dependencies resolved. 2024-09-14T08:31:55.826 INFO:teuthology.orchestra.run.smithi135.stdout:================================================================================ 2024-09-14T08:31:55.826 INFO:teuthology.orchestra.run.smithi135.stdout: Package Arch Version Repo Size 2024-09-14T08:31:55.826 INFO:teuthology.orchestra.run.smithi135.stdout:================================================================================ 2024-09-14T08:31:55.826 INFO:teuthology.orchestra.run.smithi135.stdout:Installing: 2024-09-14T08:31:55.826 INFO:teuthology.orchestra.run.smithi135.stdout: librados-devel x86_64 2:19.3.0-4775.g7f24e589.el9 ceph 126 k 2024-09-14T08:31:55.826 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:31:55.826 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction Summary 2024-09-14T08:31:55.826 INFO:teuthology.orchestra.run.smithi135.stdout:================================================================================ 2024-09-14T08:31:55.826 INFO:teuthology.orchestra.run.smithi135.stdout:Install 1 Package 2024-09-14T08:31:55.826 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:31:55.826 INFO:teuthology.orchestra.run.smithi135.stdout:Total download size: 126 k 2024-09-14T08:31:55.827 INFO:teuthology.orchestra.run.smithi135.stdout:Installed size: 456 k 2024-09-14T08:31:55.827 INFO:teuthology.orchestra.run.smithi135.stdout:Downloading Packages: 2024-09-14T08:31:56.147 INFO:teuthology.orchestra.run.smithi135.stdout:librados-devel-19.3.0-4775.g7f24e589.el9.x86_64 395 kB/s | 126 kB 00:00 2024-09-14T08:31:56.148 INFO:teuthology.orchestra.run.smithi135.stdout:-------------------------------------------------------------------------------- 2024-09-14T08:31:56.148 INFO:teuthology.orchestra.run.smithi135.stdout:Total 393 kB/s | 126 kB 00:00 2024-09-14T08:31:56.148 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction check 2024-09-14T08:31:56.154 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction check succeeded. 2024-09-14T08:31:56.154 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction test 2024-09-14T08:31:56.219 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction test succeeded. 2024-09-14T08:31:56.219 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction 2024-09-14T08:31:56.280 INFO:teuthology.orchestra.run.smithi164.stdout:(2/2): ceph-fuse-19.3.0-4775.g7f24e589.el9.x86_ 1.1 MB/s | 900 kB 00:00 2024-09-14T08:31:56.281 INFO:teuthology.orchestra.run.smithi164.stdout:-------------------------------------------------------------------------------- 2024-09-14T08:31:56.281 INFO:teuthology.orchestra.run.smithi164.stdout:Total 1.2 MB/s | 979 kB 00:00 2024-09-14T08:31:56.293 INFO:teuthology.orchestra.run.smithi164.stdout:Running transaction check 2024-09-14T08:31:56.316 INFO:teuthology.orchestra.run.smithi164.stdout:Transaction check succeeded. 2024-09-14T08:31:56.316 INFO:teuthology.orchestra.run.smithi164.stdout:Running transaction test 2024-09-14T08:31:56.369 INFO:teuthology.orchestra.run.smithi135.stdout: Preparing : 1/1 2024-09-14T08:31:56.398 INFO:teuthology.orchestra.run.smithi164.stdout:Transaction test succeeded. 2024-09-14T08:31:56.398 INFO:teuthology.orchestra.run.smithi164.stdout:Running transaction 2024-09-14T08:31:56.470 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : librados-devel-2:19.3.0-4775.g7f24e589.el9.x86_64 1/1 2024-09-14T08:31:56.584 INFO:teuthology.orchestra.run.smithi164.stdout: Preparing : 1/1 2024-09-14T08:31:56.724 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : fuse-2.9.9-16.el9.x86_64 1/2 2024-09-14T08:31:56.795 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : ceph-fuse-2:19.3.0-4775.g7f24e589.el9.x86_64 2/2 2024-09-14T08:31:56.916 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: librados-devel-2:19.3.0-4775.g7f24e589.el9.x86_64 1/1 2024-09-14T08:31:57.196 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : librados-devel-2:19.3.0-4775.g7f24e589.el9.x86_64 1/1 2024-09-14T08:31:57.196 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:31:57.196 INFO:teuthology.orchestra.run.smithi135.stdout:Installed: 2024-09-14T08:31:57.196 INFO:teuthology.orchestra.run.smithi135.stdout: librados-devel-2:19.3.0-4775.g7f24e589.el9.x86_64 2024-09-14T08:31:57.197 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:31:57.197 INFO:teuthology.orchestra.run.smithi135.stdout:Complete! 2024-09-14T08:31:57.462 DEBUG:teuthology.orchestra.run.smithi135:> sudo yum -y install libcephfs2 2024-09-14T08:31:57.531 INFO:teuthology.orchestra.run.smithi164.stdout: Running scriptlet: ceph-fuse-2:19.3.0-4775.g7f24e589.el9.x86_64 2/2 2024-09-14T08:31:57.531 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : ceph-fuse-2:19.3.0-4775.g7f24e589.el9.x86_64 1/2 2024-09-14T08:31:57.821 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : fuse-2.9.9-16.el9.x86_64 2/2 2024-09-14T08:31:57.821 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:31:57.821 INFO:teuthology.orchestra.run.smithi164.stdout:Installed: 2024-09-14T08:31:57.822 INFO:teuthology.orchestra.run.smithi164.stdout: ceph-fuse-2:19.3.0-4775.g7f24e589.el9.x86_64 fuse-2.9.9-16.el9.x86_64 2024-09-14T08:31:57.822 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:31:57.822 INFO:teuthology.orchestra.run.smithi164.stdout:Complete! 2024-09-14T08:31:58.055 INFO:teuthology.orchestra.run.smithi135.stdout:Last metadata expiration check: 0:01:41 ago on Sat 14 Sep 2024 08:30:17 AM UTC. 2024-09-14T08:31:58.098 DEBUG:teuthology.orchestra.run.smithi164:> sudo yum -y install ceph-volume 2024-09-14T08:31:58.151 INFO:teuthology.orchestra.run.smithi135.stdout:Package libcephfs2-2:19.3.0-4775.g7f24e589.el9.x86_64 is already installed. 2024-09-14T08:31:58.198 INFO:teuthology.orchestra.run.smithi135.stdout:Dependencies resolved. 2024-09-14T08:31:58.199 INFO:teuthology.orchestra.run.smithi135.stdout:Nothing to do. 2024-09-14T08:31:58.199 INFO:teuthology.orchestra.run.smithi135.stdout:Complete! 2024-09-14T08:31:58.262 DEBUG:teuthology.orchestra.run.smithi135:> sudo yum -y install libcephfs-devel 2024-09-14T08:31:58.693 INFO:teuthology.orchestra.run.smithi164.stdout:Last metadata expiration check: 0:01:42 ago on Sat 14 Sep 2024 08:30:16 AM UTC. 2024-09-14T08:31:58.836 INFO:teuthology.orchestra.run.smithi164.stdout:Dependencies resolved. 2024-09-14T08:31:58.837 INFO:teuthology.orchestra.run.smithi164.stdout:================================================================================ 2024-09-14T08:31:58.837 INFO:teuthology.orchestra.run.smithi164.stdout: Package Arch Version Repository Size 2024-09-14T08:31:58.837 INFO:teuthology.orchestra.run.smithi164.stdout:================================================================================ 2024-09-14T08:31:58.838 INFO:teuthology.orchestra.run.smithi164.stdout:Installing: 2024-09-14T08:31:58.838 INFO:teuthology.orchestra.run.smithi164.stdout: ceph-volume noarch 2:19.3.0-4775.g7f24e589.el9 ceph-noarch 285 k 2024-09-14T08:31:58.838 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:31:58.838 INFO:teuthology.orchestra.run.smithi164.stdout:Transaction Summary 2024-09-14T08:31:58.838 INFO:teuthology.orchestra.run.smithi164.stdout:================================================================================ 2024-09-14T08:31:58.838 INFO:teuthology.orchestra.run.smithi164.stdout:Install 1 Package 2024-09-14T08:31:58.838 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:31:58.838 INFO:teuthology.orchestra.run.smithi164.stdout:Total download size: 285 k 2024-09-14T08:31:58.838 INFO:teuthology.orchestra.run.smithi164.stdout:Installed size: 1.3 M 2024-09-14T08:31:58.838 INFO:teuthology.orchestra.run.smithi164.stdout:Downloading Packages: 2024-09-14T08:31:58.880 INFO:teuthology.orchestra.run.smithi135.stdout:Last metadata expiration check: 0:01:41 ago on Sat 14 Sep 2024 08:30:17 AM UTC. 2024-09-14T08:31:59.019 INFO:teuthology.orchestra.run.smithi135.stdout:Dependencies resolved. 2024-09-14T08:31:59.020 INFO:teuthology.orchestra.run.smithi135.stdout:================================================================================ 2024-09-14T08:31:59.020 INFO:teuthology.orchestra.run.smithi135.stdout: Package Arch Version Repo Size 2024-09-14T08:31:59.020 INFO:teuthology.orchestra.run.smithi135.stdout:================================================================================ 2024-09-14T08:31:59.020 INFO:teuthology.orchestra.run.smithi135.stdout:Installing: 2024-09-14T08:31:59.020 INFO:teuthology.orchestra.run.smithi135.stdout: libcephfs-devel x86_64 2:19.3.0-4775.g7f24e589.el9 ceph 32 k 2024-09-14T08:31:59.020 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:31:59.020 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction Summary 2024-09-14T08:31:59.020 INFO:teuthology.orchestra.run.smithi135.stdout:================================================================================ 2024-09-14T08:31:59.020 INFO:teuthology.orchestra.run.smithi135.stdout:Install 1 Package 2024-09-14T08:31:59.020 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:31:59.021 INFO:teuthology.orchestra.run.smithi135.stdout:Total download size: 32 k 2024-09-14T08:31:59.021 INFO:teuthology.orchestra.run.smithi135.stdout:Installed size: 142 k 2024-09-14T08:31:59.021 INFO:teuthology.orchestra.run.smithi135.stdout:Downloading Packages: 2024-09-14T08:31:59.061 INFO:teuthology.orchestra.run.smithi164.stdout:ceph-volume-19.3.0-4775.g7f24e589.el9.noarch.rp 1.3 MB/s | 285 kB 00:00 2024-09-14T08:31:59.061 INFO:teuthology.orchestra.run.smithi164.stdout:-------------------------------------------------------------------------------- 2024-09-14T08:31:59.062 INFO:teuthology.orchestra.run.smithi164.stdout:Total 1.2 MB/s | 285 kB 00:00 2024-09-14T08:31:59.062 INFO:teuthology.orchestra.run.smithi164.stdout:Running transaction check 2024-09-14T08:31:59.070 INFO:teuthology.orchestra.run.smithi164.stdout:Transaction check succeeded. 2024-09-14T08:31:59.070 INFO:teuthology.orchestra.run.smithi164.stdout:Running transaction test 2024-09-14T08:31:59.099 INFO:teuthology.orchestra.run.smithi164.stdout:Transaction test succeeded. 2024-09-14T08:31:59.100 INFO:teuthology.orchestra.run.smithi164.stdout:Running transaction 2024-09-14T08:31:59.177 INFO:teuthology.orchestra.run.smithi135.stdout:libcephfs-devel-19.3.0-4775.g7f24e589.el9.x86_6 204 kB/s | 32 kB 00:00 2024-09-14T08:31:59.177 INFO:teuthology.orchestra.run.smithi135.stdout:-------------------------------------------------------------------------------- 2024-09-14T08:31:59.178 INFO:teuthology.orchestra.run.smithi135.stdout:Total 202 kB/s | 32 kB 00:00 2024-09-14T08:31:59.178 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction check 2024-09-14T08:31:59.181 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction check succeeded. 2024-09-14T08:31:59.182 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction test 2024-09-14T08:31:59.201 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction test succeeded. 2024-09-14T08:31:59.201 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction 2024-09-14T08:31:59.293 INFO:teuthology.orchestra.run.smithi135.stdout: Preparing : 1/1 2024-09-14T08:31:59.317 INFO:teuthology.orchestra.run.smithi164.stdout: Preparing : 1/1 2024-09-14T08:31:59.380 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : ceph-volume-2:19.3.0-4775.g7f24e589.el9.noarch 1/1 2024-09-14T08:31:59.396 INFO:teuthology.orchestra.run.smithi164.stdout: Running scriptlet: ceph-volume-2:19.3.0-4775.g7f24e589.el9.noarch 1/1 2024-09-14T08:31:59.397 INFO:teuthology.orchestra.run.smithi164.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-14T08:31:59.397 INFO:teuthology.orchestra.run.smithi164.stdout:Invalid unit name "ceph-volume@*.service" escaped as "ceph-volume@\x2a.service". 2024-09-14T08:31:59.397 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:31:59.415 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : libcephfs-devel-2:19.3.0-4775.g7f24e589.el9.x86_64 1/1 2024-09-14T08:31:59.853 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: libcephfs-devel-2:19.3.0-4775.g7f24e589.el9.x86_64 1/1 2024-09-14T08:32:00.090 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : libcephfs-devel-2:19.3.0-4775.g7f24e589.el9.x86_64 1/1 2024-09-14T08:32:00.091 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:32:00.091 INFO:teuthology.orchestra.run.smithi135.stdout:Installed: 2024-09-14T08:32:00.091 INFO:teuthology.orchestra.run.smithi135.stdout: libcephfs-devel-2:19.3.0-4775.g7f24e589.el9.x86_64 2024-09-14T08:32:00.091 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:32:00.091 INFO:teuthology.orchestra.run.smithi135.stdout:Complete! 2024-09-14T08:32:00.273 DEBUG:teuthology.orchestra.run.smithi135:> sudo yum -y install librados2 2024-09-14T08:32:00.459 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : ceph-volume-2:19.3.0-4775.g7f24e589.el9.noarch 1/1 2024-09-14T08:32:00.459 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:32:00.459 INFO:teuthology.orchestra.run.smithi164.stdout:Installed: 2024-09-14T08:32:00.459 INFO:teuthology.orchestra.run.smithi164.stdout: ceph-volume-2:19.3.0-4775.g7f24e589.el9.noarch 2024-09-14T08:32:00.459 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:32:00.459 INFO:teuthology.orchestra.run.smithi164.stdout:Complete! 2024-09-14T08:32:00.753 DEBUG:teuthology.orchestra.run.smithi164:> sudo yum -y install librados-devel 2024-09-14T08:32:00.843 INFO:teuthology.orchestra.run.smithi135.stdout:Last metadata expiration check: 0:01:43 ago on Sat 14 Sep 2024 08:30:17 AM UTC. 2024-09-14T08:32:00.936 INFO:teuthology.orchestra.run.smithi135.stdout:Package librados2-2:19.3.0-4775.g7f24e589.el9.x86_64 is already installed. 2024-09-14T08:32:00.978 INFO:teuthology.orchestra.run.smithi135.stdout:Dependencies resolved. 2024-09-14T08:32:00.979 INFO:teuthology.orchestra.run.smithi135.stdout:Nothing to do. 2024-09-14T08:32:00.979 INFO:teuthology.orchestra.run.smithi135.stdout:Complete! 2024-09-14T08:32:01.054 DEBUG:teuthology.orchestra.run.smithi135:> sudo yum -y install librbd1 2024-09-14T08:32:01.336 INFO:teuthology.orchestra.run.smithi164.stdout:Last metadata expiration check: 0:01:45 ago on Sat 14 Sep 2024 08:30:16 AM UTC. 2024-09-14T08:32:01.475 INFO:teuthology.orchestra.run.smithi164.stdout:Dependencies resolved. 2024-09-14T08:32:01.475 INFO:teuthology.orchestra.run.smithi164.stdout:================================================================================ 2024-09-14T08:32:01.475 INFO:teuthology.orchestra.run.smithi164.stdout: Package Arch Version Repo Size 2024-09-14T08:32:01.475 INFO:teuthology.orchestra.run.smithi164.stdout:================================================================================ 2024-09-14T08:32:01.475 INFO:teuthology.orchestra.run.smithi164.stdout:Installing: 2024-09-14T08:32:01.475 INFO:teuthology.orchestra.run.smithi164.stdout: librados-devel x86_64 2:19.3.0-4775.g7f24e589.el9 ceph 126 k 2024-09-14T08:32:01.475 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:32:01.476 INFO:teuthology.orchestra.run.smithi164.stdout:Transaction Summary 2024-09-14T08:32:01.476 INFO:teuthology.orchestra.run.smithi164.stdout:================================================================================ 2024-09-14T08:32:01.476 INFO:teuthology.orchestra.run.smithi164.stdout:Install 1 Package 2024-09-14T08:32:01.476 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:32:01.476 INFO:teuthology.orchestra.run.smithi164.stdout:Total download size: 126 k 2024-09-14T08:32:01.476 INFO:teuthology.orchestra.run.smithi164.stdout:Installed size: 456 k 2024-09-14T08:32:01.477 INFO:teuthology.orchestra.run.smithi164.stdout:Downloading Packages: 2024-09-14T08:32:01.625 INFO:teuthology.orchestra.run.smithi135.stdout:Last metadata expiration check: 0:01:44 ago on Sat 14 Sep 2024 08:30:17 AM UTC. 2024-09-14T08:32:01.666 INFO:teuthology.orchestra.run.smithi164.stdout:librados-devel-19.3.0-4775.g7f24e589.el9.x86_64 667 kB/s | 126 kB 00:00 2024-09-14T08:32:01.667 INFO:teuthology.orchestra.run.smithi164.stdout:-------------------------------------------------------------------------------- 2024-09-14T08:32:01.667 INFO:teuthology.orchestra.run.smithi164.stdout:Total 661 kB/s | 126 kB 00:00 2024-09-14T08:32:01.667 INFO:teuthology.orchestra.run.smithi164.stdout:Running transaction check 2024-09-14T08:32:01.673 INFO:teuthology.orchestra.run.smithi164.stdout:Transaction check succeeded. 2024-09-14T08:32:01.673 INFO:teuthology.orchestra.run.smithi164.stdout:Running transaction test 2024-09-14T08:32:01.716 INFO:teuthology.orchestra.run.smithi135.stdout:Package librbd1-2:19.3.0-4775.g7f24e589.el9.x86_64 is already installed. 2024-09-14T08:32:01.737 INFO:teuthology.orchestra.run.smithi164.stdout:Transaction test succeeded. 2024-09-14T08:32:01.738 INFO:teuthology.orchestra.run.smithi164.stdout:Running transaction 2024-09-14T08:32:01.759 INFO:teuthology.orchestra.run.smithi135.stdout:Dependencies resolved. 2024-09-14T08:32:01.760 INFO:teuthology.orchestra.run.smithi135.stdout:Nothing to do. 2024-09-14T08:32:01.760 INFO:teuthology.orchestra.run.smithi135.stdout:Complete! 2024-09-14T08:32:01.836 DEBUG:teuthology.orchestra.run.smithi135:> sudo yum -y install python3-rados 2024-09-14T08:32:01.890 INFO:teuthology.orchestra.run.smithi164.stdout: Preparing : 1/1 2024-09-14T08:32:02.005 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : librados-devel-2:19.3.0-4775.g7f24e589.el9.x86_64 1/1 2024-09-14T08:32:02.406 INFO:teuthology.orchestra.run.smithi135.stdout:Last metadata expiration check: 0:01:45 ago on Sat 14 Sep 2024 08:30:17 AM UTC. 2024-09-14T08:32:02.437 INFO:teuthology.orchestra.run.smithi164.stdout: Running scriptlet: librados-devel-2:19.3.0-4775.g7f24e589.el9.x86_64 1/1 2024-09-14T08:32:02.497 INFO:teuthology.orchestra.run.smithi135.stdout:Package python3-rados-2:19.3.0-4775.g7f24e589.el9.x86_64 is already installed. 2024-09-14T08:32:02.541 INFO:teuthology.orchestra.run.smithi135.stdout:Dependencies resolved. 2024-09-14T08:32:02.542 INFO:teuthology.orchestra.run.smithi135.stdout:Nothing to do. 2024-09-14T08:32:02.542 INFO:teuthology.orchestra.run.smithi135.stdout:Complete! 2024-09-14T08:32:02.607 DEBUG:teuthology.orchestra.run.smithi135:> sudo yum -y install python3-rgw 2024-09-14T08:32:02.716 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : librados-devel-2:19.3.0-4775.g7f24e589.el9.x86_64 1/1 2024-09-14T08:32:02.717 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:32:02.717 INFO:teuthology.orchestra.run.smithi164.stdout:Installed: 2024-09-14T08:32:02.717 INFO:teuthology.orchestra.run.smithi164.stdout: librados-devel-2:19.3.0-4775.g7f24e589.el9.x86_64 2024-09-14T08:32:02.717 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:32:02.717 INFO:teuthology.orchestra.run.smithi164.stdout:Complete! 2024-09-14T08:32:02.969 DEBUG:teuthology.orchestra.run.smithi164:> sudo yum -y install libcephfs2 2024-09-14T08:32:03.177 INFO:teuthology.orchestra.run.smithi135.stdout:Last metadata expiration check: 0:01:46 ago on Sat 14 Sep 2024 08:30:17 AM UTC. 2024-09-14T08:32:03.268 INFO:teuthology.orchestra.run.smithi135.stdout:Package python3-rgw-2:19.3.0-4775.g7f24e589.el9.x86_64 is already installed. 2024-09-14T08:32:03.311 INFO:teuthology.orchestra.run.smithi135.stdout:Dependencies resolved. 2024-09-14T08:32:03.312 INFO:teuthology.orchestra.run.smithi135.stdout:Nothing to do. 2024-09-14T08:32:03.312 INFO:teuthology.orchestra.run.smithi135.stdout:Complete! 2024-09-14T08:32:03.383 DEBUG:teuthology.orchestra.run.smithi135:> sudo yum -y install python3-cephfs 2024-09-14T08:32:03.553 INFO:teuthology.orchestra.run.smithi164.stdout:Last metadata expiration check: 0:01:47 ago on Sat 14 Sep 2024 08:30:16 AM UTC. 2024-09-14T08:32:03.647 INFO:teuthology.orchestra.run.smithi164.stdout:Package libcephfs2-2:19.3.0-4775.g7f24e589.el9.x86_64 is already installed. 2024-09-14T08:32:03.692 INFO:teuthology.orchestra.run.smithi164.stdout:Dependencies resolved. 2024-09-14T08:32:03.693 INFO:teuthology.orchestra.run.smithi164.stdout:Nothing to do. 2024-09-14T08:32:03.694 INFO:teuthology.orchestra.run.smithi164.stdout:Complete! 2024-09-14T08:32:03.770 DEBUG:teuthology.orchestra.run.smithi164:> sudo yum -y install libcephfs-devel 2024-09-14T08:32:03.962 INFO:teuthology.orchestra.run.smithi135.stdout:Last metadata expiration check: 0:01:46 ago on Sat 14 Sep 2024 08:30:17 AM UTC. 2024-09-14T08:32:04.053 INFO:teuthology.orchestra.run.smithi135.stdout:Package python3-cephfs-2:19.3.0-4775.g7f24e589.el9.x86_64 is already installed. 2024-09-14T08:32:04.097 INFO:teuthology.orchestra.run.smithi135.stdout:Dependencies resolved. 2024-09-14T08:32:04.098 INFO:teuthology.orchestra.run.smithi135.stdout:Nothing to do. 2024-09-14T08:32:04.098 INFO:teuthology.orchestra.run.smithi135.stdout:Complete! 2024-09-14T08:32:04.177 DEBUG:teuthology.orchestra.run.smithi135:> sudo yum -y install python3-rbd 2024-09-14T08:32:04.341 INFO:teuthology.orchestra.run.smithi164.stdout:Last metadata expiration check: 0:01:48 ago on Sat 14 Sep 2024 08:30:16 AM UTC. 2024-09-14T08:32:04.475 INFO:teuthology.orchestra.run.smithi164.stdout:Dependencies resolved. 2024-09-14T08:32:04.476 INFO:teuthology.orchestra.run.smithi164.stdout:================================================================================ 2024-09-14T08:32:04.476 INFO:teuthology.orchestra.run.smithi164.stdout: Package Arch Version Repo Size 2024-09-14T08:32:04.476 INFO:teuthology.orchestra.run.smithi164.stdout:================================================================================ 2024-09-14T08:32:04.476 INFO:teuthology.orchestra.run.smithi164.stdout:Installing: 2024-09-14T08:32:04.476 INFO:teuthology.orchestra.run.smithi164.stdout: libcephfs-devel x86_64 2:19.3.0-4775.g7f24e589.el9 ceph 32 k 2024-09-14T08:32:04.476 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:32:04.476 INFO:teuthology.orchestra.run.smithi164.stdout:Transaction Summary 2024-09-14T08:32:04.476 INFO:teuthology.orchestra.run.smithi164.stdout:================================================================================ 2024-09-14T08:32:04.476 INFO:teuthology.orchestra.run.smithi164.stdout:Install 1 Package 2024-09-14T08:32:04.476 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:32:04.476 INFO:teuthology.orchestra.run.smithi164.stdout:Total download size: 32 k 2024-09-14T08:32:04.476 INFO:teuthology.orchestra.run.smithi164.stdout:Installed size: 142 k 2024-09-14T08:32:04.477 INFO:teuthology.orchestra.run.smithi164.stdout:Downloading Packages: 2024-09-14T08:32:04.646 INFO:teuthology.orchestra.run.smithi164.stdout:libcephfs-devel-19.3.0-4775.g7f24e589.el9.x86_6 188 kB/s | 32 kB 00:00 2024-09-14T08:32:04.647 INFO:teuthology.orchestra.run.smithi164.stdout:-------------------------------------------------------------------------------- 2024-09-14T08:32:04.647 INFO:teuthology.orchestra.run.smithi164.stdout:Total 186 kB/s | 32 kB 00:00 2024-09-14T08:32:04.647 INFO:teuthology.orchestra.run.smithi164.stdout:Running transaction check 2024-09-14T08:32:04.651 INFO:teuthology.orchestra.run.smithi164.stdout:Transaction check succeeded. 2024-09-14T08:32:04.651 INFO:teuthology.orchestra.run.smithi164.stdout:Running transaction test 2024-09-14T08:32:04.670 INFO:teuthology.orchestra.run.smithi164.stdout:Transaction test succeeded. 2024-09-14T08:32:04.671 INFO:teuthology.orchestra.run.smithi164.stdout:Running transaction 2024-09-14T08:32:04.747 INFO:teuthology.orchestra.run.smithi135.stdout:Last metadata expiration check: 0:01:47 ago on Sat 14 Sep 2024 08:30:17 AM UTC. 2024-09-14T08:32:04.762 INFO:teuthology.orchestra.run.smithi164.stdout: Preparing : 1/1 2024-09-14T08:32:04.840 INFO:teuthology.orchestra.run.smithi135.stdout:Package python3-rbd-2:19.3.0-4775.g7f24e589.el9.x86_64 is already installed. 2024-09-14T08:32:04.883 INFO:teuthology.orchestra.run.smithi135.stdout:Dependencies resolved. 2024-09-14T08:32:04.884 INFO:teuthology.orchestra.run.smithi135.stdout:Nothing to do. 2024-09-14T08:32:04.884 INFO:teuthology.orchestra.run.smithi135.stdout:Complete! 2024-09-14T08:32:04.885 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : libcephfs-devel-2:19.3.0-4775.g7f24e589.el9.x86_64 1/1 2024-09-14T08:32:04.969 DEBUG:teuthology.orchestra.run.smithi135:> sudo yum -y install rbd-fuse 2024-09-14T08:32:05.268 INFO:teuthology.orchestra.run.smithi164.stdout: Running scriptlet: libcephfs-devel-2:19.3.0-4775.g7f24e589.el9.x86_64 1/1 2024-09-14T08:32:05.515 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : libcephfs-devel-2:19.3.0-4775.g7f24e589.el9.x86_64 1/1 2024-09-14T08:32:05.515 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:32:05.516 INFO:teuthology.orchestra.run.smithi164.stdout:Installed: 2024-09-14T08:32:05.516 INFO:teuthology.orchestra.run.smithi164.stdout: libcephfs-devel-2:19.3.0-4775.g7f24e589.el9.x86_64 2024-09-14T08:32:05.516 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:32:05.516 INFO:teuthology.orchestra.run.smithi164.stdout:Complete! 2024-09-14T08:32:05.540 INFO:teuthology.orchestra.run.smithi135.stdout:Last metadata expiration check: 0:01:48 ago on Sat 14 Sep 2024 08:30:17 AM UTC. 2024-09-14T08:32:05.675 INFO:teuthology.orchestra.run.smithi135.stdout:Dependencies resolved. 2024-09-14T08:32:05.676 INFO:teuthology.orchestra.run.smithi135.stdout:================================================================================ 2024-09-14T08:32:05.676 INFO:teuthology.orchestra.run.smithi135.stdout: Package Architecture Version Repository Size 2024-09-14T08:32:05.676 INFO:teuthology.orchestra.run.smithi135.stdout:================================================================================ 2024-09-14T08:32:05.676 INFO:teuthology.orchestra.run.smithi135.stdout:Installing: 2024-09-14T08:32:05.676 INFO:teuthology.orchestra.run.smithi135.stdout: rbd-fuse x86_64 2:19.3.0-4775.g7f24e589.el9 ceph 85 k 2024-09-14T08:32:05.677 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:32:05.677 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction Summary 2024-09-14T08:32:05.677 INFO:teuthology.orchestra.run.smithi135.stdout:================================================================================ 2024-09-14T08:32:05.677 INFO:teuthology.orchestra.run.smithi135.stdout:Install 1 Package 2024-09-14T08:32:05.677 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:32:05.677 INFO:teuthology.orchestra.run.smithi135.stdout:Total download size: 85 k 2024-09-14T08:32:05.677 INFO:teuthology.orchestra.run.smithi135.stdout:Installed size: 231 k 2024-09-14T08:32:05.677 INFO:teuthology.orchestra.run.smithi135.stdout:Downloading Packages: 2024-09-14T08:32:05.705 DEBUG:teuthology.orchestra.run.smithi164:> sudo yum -y install librados2 2024-09-14T08:32:05.863 INFO:teuthology.orchestra.run.smithi135.stdout:rbd-fuse-19.3.0-4775.g7f24e589.el9.x86_64.rpm 456 kB/s | 85 kB 00:00 2024-09-14T08:32:05.864 INFO:teuthology.orchestra.run.smithi135.stdout:-------------------------------------------------------------------------------- 2024-09-14T08:32:05.864 INFO:teuthology.orchestra.run.smithi135.stdout:Total 452 kB/s | 85 kB 00:00 2024-09-14T08:32:05.865 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction check 2024-09-14T08:32:05.871 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction check succeeded. 2024-09-14T08:32:05.871 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction test 2024-09-14T08:32:05.934 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction test succeeded. 2024-09-14T08:32:05.934 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction 2024-09-14T08:32:06.079 INFO:teuthology.orchestra.run.smithi135.stdout: Preparing : 1/1 2024-09-14T08:32:06.177 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : rbd-fuse-2:19.3.0-4775.g7f24e589.el9.x86_64 1/1 2024-09-14T08:32:06.272 INFO:teuthology.orchestra.run.smithi164.stdout:Last metadata expiration check: 0:01:50 ago on Sat 14 Sep 2024 08:30:16 AM UTC. 2024-09-14T08:32:06.362 INFO:teuthology.orchestra.run.smithi164.stdout:Package librados2-2:19.3.0-4775.g7f24e589.el9.x86_64 is already installed. 2024-09-14T08:32:06.405 INFO:teuthology.orchestra.run.smithi164.stdout:Dependencies resolved. 2024-09-14T08:32:06.406 INFO:teuthology.orchestra.run.smithi164.stdout:Nothing to do. 2024-09-14T08:32:06.406 INFO:teuthology.orchestra.run.smithi164.stdout:Complete! 2024-09-14T08:32:06.477 DEBUG:teuthology.orchestra.run.smithi164:> sudo yum -y install librbd1 2024-09-14T08:32:06.602 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: rbd-fuse-2:19.3.0-4775.g7f24e589.el9.x86_64 1/1 2024-09-14T08:32:06.882 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : rbd-fuse-2:19.3.0-4775.g7f24e589.el9.x86_64 1/1 2024-09-14T08:32:06.882 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:32:06.882 INFO:teuthology.orchestra.run.smithi135.stdout:Installed: 2024-09-14T08:32:06.882 INFO:teuthology.orchestra.run.smithi135.stdout: rbd-fuse-2:19.3.0-4775.g7f24e589.el9.x86_64 2024-09-14T08:32:06.883 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:32:06.883 INFO:teuthology.orchestra.run.smithi135.stdout:Complete! 2024-09-14T08:32:07.042 INFO:teuthology.orchestra.run.smithi164.stdout:Last metadata expiration check: 0:01:51 ago on Sat 14 Sep 2024 08:30:16 AM UTC. 2024-09-14T08:32:07.133 INFO:teuthology.orchestra.run.smithi164.stdout:Package librbd1-2:19.3.0-4775.g7f24e589.el9.x86_64 is already installed. 2024-09-14T08:32:07.177 DEBUG:teuthology.orchestra.run.smithi135:> sudo yum -y install rbd-mirror 2024-09-14T08:32:07.178 INFO:teuthology.orchestra.run.smithi164.stdout:Dependencies resolved. 2024-09-14T08:32:07.178 INFO:teuthology.orchestra.run.smithi164.stdout:Nothing to do. 2024-09-14T08:32:07.178 INFO:teuthology.orchestra.run.smithi164.stdout:Complete! 2024-09-14T08:32:07.249 DEBUG:teuthology.orchestra.run.smithi164:> sudo yum -y install python3-rados 2024-09-14T08:32:07.760 INFO:teuthology.orchestra.run.smithi135.stdout:Last metadata expiration check: 0:01:50 ago on Sat 14 Sep 2024 08:30:17 AM UTC. 2024-09-14T08:32:07.861 INFO:teuthology.orchestra.run.smithi164.stdout:Last metadata expiration check: 0:01:51 ago on Sat 14 Sep 2024 08:30:16 AM UTC. 2024-09-14T08:32:07.901 INFO:teuthology.orchestra.run.smithi135.stdout:Dependencies resolved. 2024-09-14T08:32:07.902 INFO:teuthology.orchestra.run.smithi135.stdout:================================================================================ 2024-09-14T08:32:07.902 INFO:teuthology.orchestra.run.smithi135.stdout: Package Arch Version Repo Size 2024-09-14T08:32:07.902 INFO:teuthology.orchestra.run.smithi135.stdout:================================================================================ 2024-09-14T08:32:07.902 INFO:teuthology.orchestra.run.smithi135.stdout:Installing: 2024-09-14T08:32:07.902 INFO:teuthology.orchestra.run.smithi135.stdout: rbd-mirror x86_64 2:19.3.0-4775.g7f24e589.el9 ceph 3.2 M 2024-09-14T08:32:07.902 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:32:07.902 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction Summary 2024-09-14T08:32:07.902 INFO:teuthology.orchestra.run.smithi135.stdout:================================================================================ 2024-09-14T08:32:07.902 INFO:teuthology.orchestra.run.smithi135.stdout:Install 1 Package 2024-09-14T08:32:07.902 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:32:07.902 INFO:teuthology.orchestra.run.smithi135.stdout:Total download size: 3.2 M 2024-09-14T08:32:07.902 INFO:teuthology.orchestra.run.smithi135.stdout:Installed size: 14 M 2024-09-14T08:32:07.903 INFO:teuthology.orchestra.run.smithi135.stdout:Downloading Packages: 2024-09-14T08:32:07.953 INFO:teuthology.orchestra.run.smithi164.stdout:Package python3-rados-2:19.3.0-4775.g7f24e589.el9.x86_64 is already installed. 2024-09-14T08:32:07.996 INFO:teuthology.orchestra.run.smithi164.stdout:Dependencies resolved. 2024-09-14T08:32:07.998 INFO:teuthology.orchestra.run.smithi164.stdout:Nothing to do. 2024-09-14T08:32:07.998 INFO:teuthology.orchestra.run.smithi164.stdout:Complete! 2024-09-14T08:32:08.068 DEBUG:teuthology.orchestra.run.smithi164:> sudo yum -y install python3-rgw 2024-09-14T08:32:08.233 INFO:teuthology.orchestra.run.smithi135.stdout:rbd-mirror-19.3.0-4775.g7f24e589.el9.x86_64.rpm 9.6 MB/s | 3.2 MB 00:00 2024-09-14T08:32:08.234 INFO:teuthology.orchestra.run.smithi135.stdout:-------------------------------------------------------------------------------- 2024-09-14T08:32:08.234 INFO:teuthology.orchestra.run.smithi135.stdout:Total 9.6 MB/s | 3.2 MB 00:00 2024-09-14T08:32:08.235 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction check 2024-09-14T08:32:08.244 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction check succeeded. 2024-09-14T08:32:08.244 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction test 2024-09-14T08:32:08.325 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction test succeeded. 2024-09-14T08:32:08.325 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction 2024-09-14T08:32:08.643 INFO:teuthology.orchestra.run.smithi164.stdout:Last metadata expiration check: 0:01:52 ago on Sat 14 Sep 2024 08:30:16 AM UTC. 2024-09-14T08:32:08.733 INFO:teuthology.orchestra.run.smithi164.stdout:Package python3-rgw-2:19.3.0-4775.g7f24e589.el9.x86_64 is already installed. 2024-09-14T08:32:08.767 INFO:teuthology.orchestra.run.smithi135.stdout: Preparing : 1/1 2024-09-14T08:32:08.776 INFO:teuthology.orchestra.run.smithi164.stdout:Dependencies resolved. 2024-09-14T08:32:08.778 INFO:teuthology.orchestra.run.smithi164.stdout:Nothing to do. 2024-09-14T08:32:08.778 INFO:teuthology.orchestra.run.smithi164.stdout:Complete! 2024-09-14T08:32:08.843 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : rbd-mirror-2:19.3.0-4775.g7f24e589.el9.x86_64 1/1 2024-09-14T08:32:08.858 DEBUG:teuthology.orchestra.run.smithi164:> sudo yum -y install python3-cephfs 2024-09-14T08:32:08.869 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: rbd-mirror-2:19.3.0-4775.g7f24e589.el9.x86_64 1/1 2024-09-14T08:32:08.869 INFO:teuthology.orchestra.run.smithi135.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-14T08:32:08.869 INFO:teuthology.orchestra.run.smithi135.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2024-09-14T08:32:08.869 INFO:teuthology.orchestra.run.smithi135.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-09-14T08:32:08.869 INFO:teuthology.orchestra.run.smithi135.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-09-14T08:32:08.869 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:32:09.430 INFO:teuthology.orchestra.run.smithi164.stdout:Last metadata expiration check: 0:01:53 ago on Sat 14 Sep 2024 08:30:16 AM UTC. 2024-09-14T08:32:09.523 INFO:teuthology.orchestra.run.smithi164.stdout:Package python3-cephfs-2:19.3.0-4775.g7f24e589.el9.x86_64 is already installed. 2024-09-14T08:32:09.567 INFO:teuthology.orchestra.run.smithi164.stdout:Dependencies resolved. 2024-09-14T08:32:09.568 INFO:teuthology.orchestra.run.smithi164.stdout:Nothing to do. 2024-09-14T08:32:09.568 INFO:teuthology.orchestra.run.smithi164.stdout:Complete! 2024-09-14T08:32:09.643 DEBUG:teuthology.orchestra.run.smithi164:> sudo yum -y install python3-rbd 2024-09-14T08:32:10.031 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : rbd-mirror-2:19.3.0-4775.g7f24e589.el9.x86_64 1/1 2024-09-14T08:32:10.031 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:32:10.031 INFO:teuthology.orchestra.run.smithi135.stdout:Installed: 2024-09-14T08:32:10.031 INFO:teuthology.orchestra.run.smithi135.stdout: rbd-mirror-2:19.3.0-4775.g7f24e589.el9.x86_64 2024-09-14T08:32:10.032 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:32:10.032 INFO:teuthology.orchestra.run.smithi135.stdout:Complete! 2024-09-14T08:32:10.221 INFO:teuthology.orchestra.run.smithi164.stdout:Last metadata expiration check: 0:01:54 ago on Sat 14 Sep 2024 08:30:16 AM UTC. 2024-09-14T08:32:10.297 DEBUG:teuthology.orchestra.run.smithi135:> sudo yum -y install rbd-nbd 2024-09-14T08:32:10.311 INFO:teuthology.orchestra.run.smithi164.stdout:Package python3-rbd-2:19.3.0-4775.g7f24e589.el9.x86_64 is already installed. 2024-09-14T08:32:10.356 INFO:teuthology.orchestra.run.smithi164.stdout:Dependencies resolved. 2024-09-14T08:32:10.357 INFO:teuthology.orchestra.run.smithi164.stdout:Nothing to do. 2024-09-14T08:32:10.357 INFO:teuthology.orchestra.run.smithi164.stdout:Complete! 2024-09-14T08:32:10.431 DEBUG:teuthology.orchestra.run.smithi164:> sudo yum -y install rbd-fuse 2024-09-14T08:32:10.880 INFO:teuthology.orchestra.run.smithi135.stdout:Last metadata expiration check: 0:01:53 ago on Sat 14 Sep 2024 08:30:17 AM UTC. 2024-09-14T08:32:10.999 INFO:teuthology.orchestra.run.smithi164.stdout:Last metadata expiration check: 0:01:54 ago on Sat 14 Sep 2024 08:30:16 AM UTC. 2024-09-14T08:32:11.015 INFO:teuthology.orchestra.run.smithi135.stdout:Dependencies resolved. 2024-09-14T08:32:11.015 INFO:teuthology.orchestra.run.smithi135.stdout:================================================================================ 2024-09-14T08:32:11.015 INFO:teuthology.orchestra.run.smithi135.stdout: Package Architecture Version Repository Size 2024-09-14T08:32:11.015 INFO:teuthology.orchestra.run.smithi135.stdout:================================================================================ 2024-09-14T08:32:11.015 INFO:teuthology.orchestra.run.smithi135.stdout:Installing: 2024-09-14T08:32:11.016 INFO:teuthology.orchestra.run.smithi135.stdout: rbd-nbd x86_64 2:19.3.0-4775.g7f24e589.el9 ceph 171 k 2024-09-14T08:32:11.016 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:32:11.016 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction Summary 2024-09-14T08:32:11.016 INFO:teuthology.orchestra.run.smithi135.stdout:================================================================================ 2024-09-14T08:32:11.016 INFO:teuthology.orchestra.run.smithi135.stdout:Install 1 Package 2024-09-14T08:32:11.016 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:32:11.016 INFO:teuthology.orchestra.run.smithi135.stdout:Total download size: 171 k 2024-09-14T08:32:11.017 INFO:teuthology.orchestra.run.smithi135.stdout:Installed size: 490 k 2024-09-14T08:32:11.017 INFO:teuthology.orchestra.run.smithi135.stdout:Downloading Packages: 2024-09-14T08:32:11.135 INFO:teuthology.orchestra.run.smithi164.stdout:Dependencies resolved. 2024-09-14T08:32:11.136 INFO:teuthology.orchestra.run.smithi164.stdout:================================================================================ 2024-09-14T08:32:11.136 INFO:teuthology.orchestra.run.smithi164.stdout: Package Architecture Version Repository Size 2024-09-14T08:32:11.136 INFO:teuthology.orchestra.run.smithi164.stdout:================================================================================ 2024-09-14T08:32:11.136 INFO:teuthology.orchestra.run.smithi164.stdout:Installing: 2024-09-14T08:32:11.136 INFO:teuthology.orchestra.run.smithi164.stdout: rbd-fuse x86_64 2:19.3.0-4775.g7f24e589.el9 ceph 85 k 2024-09-14T08:32:11.136 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:32:11.136 INFO:teuthology.orchestra.run.smithi164.stdout:Transaction Summary 2024-09-14T08:32:11.136 INFO:teuthology.orchestra.run.smithi164.stdout:================================================================================ 2024-09-14T08:32:11.136 INFO:teuthology.orchestra.run.smithi164.stdout:Install 1 Package 2024-09-14T08:32:11.136 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:32:11.137 INFO:teuthology.orchestra.run.smithi164.stdout:Total download size: 85 k 2024-09-14T08:32:11.137 INFO:teuthology.orchestra.run.smithi164.stdout:Installed size: 231 k 2024-09-14T08:32:11.137 INFO:teuthology.orchestra.run.smithi164.stdout:Downloading Packages: 2024-09-14T08:32:11.332 INFO:teuthology.orchestra.run.smithi164.stdout:rbd-fuse-19.3.0-4775.g7f24e589.el9.x86_64.rpm 436 kB/s | 85 kB 00:00 2024-09-14T08:32:11.333 INFO:teuthology.orchestra.run.smithi164.stdout:-------------------------------------------------------------------------------- 2024-09-14T08:32:11.333 INFO:teuthology.orchestra.run.smithi164.stdout:Total 433 kB/s | 85 kB 00:00 2024-09-14T08:32:11.333 INFO:teuthology.orchestra.run.smithi164.stdout:Running transaction check 2024-09-14T08:32:11.340 INFO:teuthology.orchestra.run.smithi164.stdout:Transaction check succeeded. 2024-09-14T08:32:11.340 INFO:teuthology.orchestra.run.smithi164.stdout:Running transaction test 2024-09-14T08:32:11.403 INFO:teuthology.orchestra.run.smithi164.stdout:Transaction test succeeded. 2024-09-14T08:32:11.403 INFO:teuthology.orchestra.run.smithi164.stdout:Running transaction 2024-09-14T08:32:11.547 INFO:teuthology.orchestra.run.smithi164.stdout: Preparing : 1/1 2024-09-14T08:32:11.646 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : rbd-fuse-2:19.3.0-4775.g7f24e589.el9.x86_64 1/1 2024-09-14T08:32:12.064 INFO:teuthology.orchestra.run.smithi164.stdout: Running scriptlet: rbd-fuse-2:19.3.0-4775.g7f24e589.el9.x86_64 1/1 2024-09-14T08:32:12.348 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : rbd-fuse-2:19.3.0-4775.g7f24e589.el9.x86_64 1/1 2024-09-14T08:32:12.348 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:32:12.348 INFO:teuthology.orchestra.run.smithi164.stdout:Installed: 2024-09-14T08:32:12.348 INFO:teuthology.orchestra.run.smithi164.stdout: rbd-fuse-2:19.3.0-4775.g7f24e589.el9.x86_64 2024-09-14T08:32:12.348 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:32:12.349 INFO:teuthology.orchestra.run.smithi164.stdout:Complete! 2024-09-14T08:32:12.627 DEBUG:teuthology.orchestra.run.smithi164:> sudo yum -y install rbd-mirror 2024-09-14T08:32:13.208 INFO:teuthology.orchestra.run.smithi164.stdout:Last metadata expiration check: 0:01:57 ago on Sat 14 Sep 2024 08:30:16 AM UTC. 2024-09-14T08:32:13.348 INFO:teuthology.orchestra.run.smithi164.stdout:Dependencies resolved. 2024-09-14T08:32:13.349 INFO:teuthology.orchestra.run.smithi164.stdout:================================================================================ 2024-09-14T08:32:13.349 INFO:teuthology.orchestra.run.smithi164.stdout: Package Arch Version Repo Size 2024-09-14T08:32:13.349 INFO:teuthology.orchestra.run.smithi164.stdout:================================================================================ 2024-09-14T08:32:13.349 INFO:teuthology.orchestra.run.smithi164.stdout:Installing: 2024-09-14T08:32:13.349 INFO:teuthology.orchestra.run.smithi164.stdout: rbd-mirror x86_64 2:19.3.0-4775.g7f24e589.el9 ceph 3.2 M 2024-09-14T08:32:13.349 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:32:13.349 INFO:teuthology.orchestra.run.smithi164.stdout:Transaction Summary 2024-09-14T08:32:13.349 INFO:teuthology.orchestra.run.smithi164.stdout:================================================================================ 2024-09-14T08:32:13.350 INFO:teuthology.orchestra.run.smithi164.stdout:Install 1 Package 2024-09-14T08:32:13.350 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:32:13.350 INFO:teuthology.orchestra.run.smithi164.stdout:Total download size: 3.2 M 2024-09-14T08:32:13.350 INFO:teuthology.orchestra.run.smithi164.stdout:Installed size: 14 M 2024-09-14T08:32:13.350 INFO:teuthology.orchestra.run.smithi164.stdout:Downloading Packages: 2024-09-14T08:32:13.404 INFO:teuthology.orchestra.run.smithi135.stdout:rbd-nbd-19.3.0-4775.g7f24e589.el9.x86_64.rpm 71 kB/s | 171 kB 00:02 2024-09-14T08:32:13.405 INFO:teuthology.orchestra.run.smithi135.stdout:-------------------------------------------------------------------------------- 2024-09-14T08:32:13.406 INFO:teuthology.orchestra.run.smithi135.stdout:Total 71 kB/s | 171 kB 00:02 2024-09-14T08:32:13.406 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction check 2024-09-14T08:32:13.415 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction check succeeded. 2024-09-14T08:32:13.415 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction test 2024-09-14T08:32:13.483 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction test succeeded. 2024-09-14T08:32:13.483 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction 2024-09-14T08:32:13.647 INFO:teuthology.orchestra.run.smithi135.stdout: Preparing : 1/1 2024-09-14T08:32:13.704 INFO:teuthology.orchestra.run.smithi164.stdout:rbd-mirror-19.3.0-4775.g7f24e589.el9.x86_64.rpm 9.0 MB/s | 3.2 MB 00:00 2024-09-14T08:32:13.705 INFO:teuthology.orchestra.run.smithi164.stdout:-------------------------------------------------------------------------------- 2024-09-14T08:32:13.705 INFO:teuthology.orchestra.run.smithi164.stdout:Total 8.9 MB/s | 3.2 MB 00:00 2024-09-14T08:32:13.705 INFO:teuthology.orchestra.run.smithi164.stdout:Running transaction check 2024-09-14T08:32:13.714 INFO:teuthology.orchestra.run.smithi164.stdout:Transaction check succeeded. 2024-09-14T08:32:13.714 INFO:teuthology.orchestra.run.smithi164.stdout:Running transaction test 2024-09-14T08:32:13.734 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : rbd-nbd-2:19.3.0-4775.g7f24e589.el9.x86_64 1/1 2024-09-14T08:32:13.796 INFO:teuthology.orchestra.run.smithi164.stdout:Transaction test succeeded. 2024-09-14T08:32:13.796 INFO:teuthology.orchestra.run.smithi164.stdout:Running transaction 2024-09-14T08:32:14.205 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: rbd-nbd-2:19.3.0-4775.g7f24e589.el9.x86_64 1/1 2024-09-14T08:32:14.235 INFO:teuthology.orchestra.run.smithi164.stdout: Preparing : 1/1 2024-09-14T08:32:14.312 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : rbd-mirror-2:19.3.0-4775.g7f24e589.el9.x86_64 1/1 2024-09-14T08:32:14.337 INFO:teuthology.orchestra.run.smithi164.stdout: Running scriptlet: rbd-mirror-2:19.3.0-4775.g7f24e589.el9.x86_64 1/1 2024-09-14T08:32:14.337 INFO:teuthology.orchestra.run.smithi164.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-14T08:32:14.337 INFO:teuthology.orchestra.run.smithi164.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2024-09-14T08:32:14.337 INFO:teuthology.orchestra.run.smithi164.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-09-14T08:32:14.338 INFO:teuthology.orchestra.run.smithi164.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-09-14T08:32:14.338 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:32:14.465 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : rbd-nbd-2:19.3.0-4775.g7f24e589.el9.x86_64 1/1 2024-09-14T08:32:14.465 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:32:14.465 INFO:teuthology.orchestra.run.smithi135.stdout:Installed: 2024-09-14T08:32:14.465 INFO:teuthology.orchestra.run.smithi135.stdout: rbd-nbd-2:19.3.0-4775.g7f24e589.el9.x86_64 2024-09-14T08:32:14.465 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:32:14.465 INFO:teuthology.orchestra.run.smithi135.stdout:Complete! 2024-09-14T08:32:14.690 DEBUG:teuthology.orchestra.run.smithi135:> sudo yum -y install rbd-mirror 2024-09-14T08:32:15.275 INFO:teuthology.orchestra.run.smithi135.stdout:Last metadata expiration check: 0:01:58 ago on Sat 14 Sep 2024 08:30:17 AM UTC. 2024-09-14T08:32:15.369 INFO:teuthology.orchestra.run.smithi135.stdout:Package rbd-mirror-2:19.3.0-4775.g7f24e589.el9.x86_64 is already installed. 2024-09-14T08:32:15.414 INFO:teuthology.orchestra.run.smithi135.stdout:Dependencies resolved. 2024-09-14T08:32:15.415 INFO:teuthology.orchestra.run.smithi135.stdout:Nothing to do. 2024-09-14T08:32:15.415 INFO:teuthology.orchestra.run.smithi135.stdout:Complete! 2024-09-14T08:32:15.489 DEBUG:teuthology.parallel:result is None 2024-09-14T08:32:15.490 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : rbd-mirror-2:19.3.0-4775.g7f24e589.el9.x86_64 1/1 2024-09-14T08:32:15.490 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:32:15.490 INFO:teuthology.orchestra.run.smithi164.stdout:Installed: 2024-09-14T08:32:15.490 INFO:teuthology.orchestra.run.smithi164.stdout: rbd-mirror-2:19.3.0-4775.g7f24e589.el9.x86_64 2024-09-14T08:32:15.490 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:32:15.490 INFO:teuthology.orchestra.run.smithi164.stdout:Complete! 2024-09-14T08:32:15.727 DEBUG:teuthology.orchestra.run.smithi164:> sudo yum -y install rbd-nbd 2024-09-14T08:32:16.308 INFO:teuthology.orchestra.run.smithi164.stdout:Last metadata expiration check: 0:02:00 ago on Sat 14 Sep 2024 08:30:16 AM UTC. 2024-09-14T08:32:16.443 INFO:teuthology.orchestra.run.smithi164.stdout:Dependencies resolved. 2024-09-14T08:32:16.444 INFO:teuthology.orchestra.run.smithi164.stdout:================================================================================ 2024-09-14T08:32:16.444 INFO:teuthology.orchestra.run.smithi164.stdout: Package Architecture Version Repository Size 2024-09-14T08:32:16.444 INFO:teuthology.orchestra.run.smithi164.stdout:================================================================================ 2024-09-14T08:32:16.444 INFO:teuthology.orchestra.run.smithi164.stdout:Installing: 2024-09-14T08:32:16.444 INFO:teuthology.orchestra.run.smithi164.stdout: rbd-nbd x86_64 2:19.3.0-4775.g7f24e589.el9 ceph 171 k 2024-09-14T08:32:16.444 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:32:16.444 INFO:teuthology.orchestra.run.smithi164.stdout:Transaction Summary 2024-09-14T08:32:16.444 INFO:teuthology.orchestra.run.smithi164.stdout:================================================================================ 2024-09-14T08:32:16.444 INFO:teuthology.orchestra.run.smithi164.stdout:Install 1 Package 2024-09-14T08:32:16.444 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:32:16.445 INFO:teuthology.orchestra.run.smithi164.stdout:Total download size: 171 k 2024-09-14T08:32:16.445 INFO:teuthology.orchestra.run.smithi164.stdout:Installed size: 490 k 2024-09-14T08:32:16.445 INFO:teuthology.orchestra.run.smithi164.stdout:Downloading Packages: 2024-09-14T08:32:16.650 INFO:teuthology.orchestra.run.smithi164.stdout:rbd-nbd-19.3.0-4775.g7f24e589.el9.x86_64.rpm 833 kB/s | 171 kB 00:00 2024-09-14T08:32:16.651 INFO:teuthology.orchestra.run.smithi164.stdout:-------------------------------------------------------------------------------- 2024-09-14T08:32:16.651 INFO:teuthology.orchestra.run.smithi164.stdout:Total 828 kB/s | 171 kB 00:00 2024-09-14T08:32:16.651 INFO:teuthology.orchestra.run.smithi164.stdout:Running transaction check 2024-09-14T08:32:16.659 INFO:teuthology.orchestra.run.smithi164.stdout:Transaction check succeeded. 2024-09-14T08:32:16.659 INFO:teuthology.orchestra.run.smithi164.stdout:Running transaction test 2024-09-14T08:32:16.723 INFO:teuthology.orchestra.run.smithi164.stdout:Transaction test succeeded. 2024-09-14T08:32:16.723 INFO:teuthology.orchestra.run.smithi164.stdout:Running transaction 2024-09-14T08:32:16.887 INFO:teuthology.orchestra.run.smithi164.stdout: Preparing : 1/1 2024-09-14T08:32:16.980 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : rbd-nbd-2:19.3.0-4775.g7f24e589.el9.x86_64 1/1 2024-09-14T08:32:17.423 INFO:teuthology.orchestra.run.smithi164.stdout: Running scriptlet: rbd-nbd-2:19.3.0-4775.g7f24e589.el9.x86_64 1/1 2024-09-14T08:32:17.697 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : rbd-nbd-2:19.3.0-4775.g7f24e589.el9.x86_64 1/1 2024-09-14T08:32:17.697 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:32:17.697 INFO:teuthology.orchestra.run.smithi164.stdout:Installed: 2024-09-14T08:32:17.697 INFO:teuthology.orchestra.run.smithi164.stdout: rbd-nbd-2:19.3.0-4775.g7f24e589.el9.x86_64 2024-09-14T08:32:17.697 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:32:17.697 INFO:teuthology.orchestra.run.smithi164.stdout:Complete! 2024-09-14T08:32:17.958 DEBUG:teuthology.orchestra.run.smithi164:> sudo yum -y install rbd-mirror 2024-09-14T08:32:18.534 INFO:teuthology.orchestra.run.smithi164.stdout:Last metadata expiration check: 0:02:02 ago on Sat 14 Sep 2024 08:30:16 AM UTC. 2024-09-14T08:32:18.624 INFO:teuthology.orchestra.run.smithi164.stdout:Package rbd-mirror-2:19.3.0-4775.g7f24e589.el9.x86_64 is already installed. 2024-09-14T08:32:18.668 INFO:teuthology.orchestra.run.smithi164.stdout:Dependencies resolved. 2024-09-14T08:32:18.669 INFO:teuthology.orchestra.run.smithi164.stdout:Nothing to do. 2024-09-14T08:32:18.669 INFO:teuthology.orchestra.run.smithi164.stdout:Complete! 2024-09-14T08:32:18.739 DEBUG:teuthology.parallel:result is None 2024-09-14T08:32:18.740 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=7f24e589328ae9e639c0009d93340e5b235d0a47 2024-09-14T08:32:18.886 DEBUG:teuthology.orchestra.run.smithi135:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2024-09-14T08:32:18.912 INFO:teuthology.orchestra.run.smithi135.stdout:19.3.0-4775.g7f24e589.el9 2024-09-14T08:32:18.912 INFO:teuthology.packaging:The installed version of ceph is 19.3.0-4775.g7f24e589.el9 2024-09-14T08:32:18.913 INFO:teuthology.task.install:The correct ceph version 19.3.0-4775.g7f24e589 is installed. 2024-09-14T08:32:18.914 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=7f24e589328ae9e639c0009d93340e5b235d0a47 2024-09-14T08:32:19.067 DEBUG:teuthology.orchestra.run.smithi164:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2024-09-14T08:32:19.088 INFO:teuthology.orchestra.run.smithi164.stdout:19.3.0-4775.g7f24e589.el9 2024-09-14T08:32:19.088 INFO:teuthology.packaging:The installed version of ceph is 19.3.0-4775.g7f24e589.el9 2024-09-14T08:32:19.088 INFO:teuthology.task.install:The correct ceph version 19.3.0-4775.g7f24e589 is installed. 2024-09-14T08:32:19.090 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2024-09-14T08:32:19.090 DEBUG:teuthology.orchestra.run.smithi135:> set -ex 2024-09-14T08:32:19.090 DEBUG:teuthology.orchestra.run.smithi135:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-09-14T08:32:19.120 DEBUG:teuthology.orchestra.run.smithi164:> set -ex 2024-09-14T08:32:19.120 DEBUG:teuthology.orchestra.run.smithi164:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-09-14T08:32:19.154 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2024-09-14T08:32:19.155 DEBUG:teuthology.orchestra.run.smithi135:> set -ex 2024-09-14T08:32:19.155 DEBUG:teuthology.orchestra.run.smithi135:> sudo dd of=/usr/bin/daemon-helper 2024-09-14T08:32:19.190 DEBUG:teuthology.orchestra.run.smithi135:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-09-14T08:32:19.259 DEBUG:teuthology.orchestra.run.smithi164:> set -ex 2024-09-14T08:32:19.259 DEBUG:teuthology.orchestra.run.smithi164:> sudo dd of=/usr/bin/daemon-helper 2024-09-14T08:32:19.285 DEBUG:teuthology.orchestra.run.smithi164:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-09-14T08:32:19.350 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2024-09-14T08:32:19.350 DEBUG:teuthology.orchestra.run.smithi135:> set -ex 2024-09-14T08:32:19.350 DEBUG:teuthology.orchestra.run.smithi135:> sudo dd of=/usr/bin/adjust-ulimits 2024-09-14T08:32:19.378 DEBUG:teuthology.orchestra.run.smithi135:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-09-14T08:32:19.445 DEBUG:teuthology.orchestra.run.smithi164:> set -ex 2024-09-14T08:32:19.445 DEBUG:teuthology.orchestra.run.smithi164:> sudo dd of=/usr/bin/adjust-ulimits 2024-09-14T08:32:19.470 DEBUG:teuthology.orchestra.run.smithi164:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-09-14T08:32:19.535 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2024-09-14T08:32:19.535 DEBUG:teuthology.orchestra.run.smithi135:> set -ex 2024-09-14T08:32:19.535 DEBUG:teuthology.orchestra.run.smithi135:> sudo dd of=/usr/bin/stdin-killer 2024-09-14T08:32:19.562 DEBUG:teuthology.orchestra.run.smithi135:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-09-14T08:32:19.628 DEBUG:teuthology.orchestra.run.smithi164:> set -ex 2024-09-14T08:32:19.628 DEBUG:teuthology.orchestra.run.smithi164:> sudo dd of=/usr/bin/stdin-killer 2024-09-14T08:32:19.653 DEBUG:teuthology.orchestra.run.smithi164:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-09-14T08:32:19.718 INFO:teuthology.run_tasks:Running task ceph... 2024-09-14T08:32:19.795 INFO:tasks.ceph:Making ceph log dir writeable by non-root... 2024-09-14T08:32:19.795 DEBUG:teuthology.orchestra.run.smithi135:> sudo chmod 777 /var/log/ceph 2024-09-14T08:32:19.798 DEBUG:teuthology.orchestra.run.smithi164:> sudo chmod 777 /var/log/ceph 2024-09-14T08:32:19.822 INFO:tasks.ceph:Disabling ceph logrotate... 2024-09-14T08:32:19.822 DEBUG:teuthology.orchestra.run.smithi135:> sudo rm -f -- /etc/logrotate.d/ceph 2024-09-14T08:32:19.864 DEBUG:teuthology.orchestra.run.smithi164:> sudo rm -f -- /etc/logrotate.d/ceph 2024-09-14T08:32:19.888 INFO:tasks.ceph:Creating extra log directories... 2024-09-14T08:32:19.888 DEBUG:teuthology.orchestra.run.smithi135:> sudo install -d -m0777 -- /var/log/ceph/valgrind /var/log/ceph/profiling-logger 2024-09-14T08:32:19.929 DEBUG:teuthology.orchestra.run.smithi164:> sudo install -d -m0777 -- /var/log/ceph/valgrind /var/log/ceph/profiling-logger 2024-09-14T08:32:19.966 INFO:tasks.ceph:Creating ceph cluster cluster1... 2024-09-14T08:32:19.966 INFO:tasks.ceph:config {'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'simple'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bluestore block size': 96636764160, 'bluestore compression algorithm': 'zlib', 'bluestore compression mode': 'aggressive', 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'fs': 'xfs', 'mkfs_options': None, 'mount_options': None, 'skip_mgr_daemons': False, 'log_ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'cpu_profile': set(), 'cluster': 'cluster1', 'mon_bind_msgr2': True, 'mon_bind_addrvec': True} 2024-09-14T08:32:19.966 INFO:tasks.ceph:ctx.config {'archive_path': '/home/teuthworker/archive/teuthology-2024-09-10_20:16:17-rbd-main-distro-default-smithi/7900492', 'branch': 'main', 'description': 'rbd/mirror-thrash/{base/install clients/mirror cluster/{2-node openstack} conf/{disable-pool-app} msgr-failures/few objectstore/bluestore-comp-zlib policy/simple rbd-mirror/four-per-cluster supported-random-distro$/{centos_latest} workloads/rbd-mirror-snapshot-stress-workunit-minimum}', 'email': 'ceph-qa@ceph.com', 'first_in_suite': False, 'job_id': '7900492', 'kernel': {'kdb': 1, 'sha1': 'distro'}, 'last_in_suite': False, 'machine_type': 'smithi', 'meta': [{'desc': 'run two ceph clusters and install rbd-mirror'}, {'desc': 'configure the permissions for client.mirror'}, {'desc': '2 ceph clusters with 1 mon and 3 osds each'}, {'desc': 'run four rbd-mirror daemons per cluster'}, {'desc': 'run the rbd_mirror_stress.sh workunit to test the rbd-mirror daemon'}], 'name': 'teuthology-2024-09-10_20:16:17-rbd-main-distro-default-smithi', 'no_nested_subset': False, 'openstack': [{'volumes': {'count': 3, 'size': 30}}], 'os_type': 'centos', 'os_version': '9.stream', 'overrides': {'admin_socket': {'branch': 'main'}, 'ceph': {'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'simple'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bluestore block size': 96636764160, 'bluestore compression algorithm': 'zlib', 'bluestore compression mode': 'aggressive', 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': '7f24e589328ae9e639c0009d93340e5b235d0a47'}, 'ceph-deploy': {'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}}}, 'install': {'ceph': {'flavor': 'default', 'sha1': '7f24e589328ae9e639c0009d93340e5b235d0a47'}}, 'thrashosds': {'bdev_inject_crash': 2, 'bdev_inject_crash_probability': 0.5}, 'workunit': {'branch': 'main', 'sha1': '0d61375b8ab53be4c233f5b93e463edfdff9316a'}}, 'owner': 'scheduled_teuthology@teuthology', 'priority': 950, 'repo': 'https://git.ceph.com/ceph.git', 'roles': [['cluster1.mon.a', 'cluster1.mgr.x', 'cluster2.mgr.x', 'cluster1.osd.0', 'cluster1.osd.1', 'cluster1.osd.2', 'cluster1.client.0', 'cluster2.client.0'], ['cluster2.mon.a', 'cluster2.osd.0', 'cluster2.osd.1', 'cluster2.osd.2', 'cluster1.client.mirror', 'cluster1.client.mirror.0', 'cluster1.client.mirror.1', 'cluster1.client.mirror.2', 'cluster1.client.mirror.3', 'cluster1.client.mirror.4', 'cluster1.client.mirror.5', 'cluster1.client.mirror.6', 'cluster2.client.mirror', 'cluster2.client.mirror.0', 'cluster2.client.mirror.1', 'cluster2.client.mirror.2', 'cluster2.client.mirror.3', 'cluster2.client.mirror.4', 'cluster2.client.mirror.5', 'cluster2.client.mirror.6']], 'seed': 5839, 'sha1': '7f24e589328ae9e639c0009d93340e5b235d0a47', 'sleep_before_teardown': 0, 'subset': '99/128', 'suite': 'rbd', 'suite_branch': 'main', 'suite_path': '/home/teuthworker/src/git.ceph.com_ceph_0d61375b8ab53be4c233f5b93e463edfdff9316a/qa', 'suite_relpath': 'qa', 'suite_repo': 'https://git.ceph.com/ceph.git', 'suite_sha1': '0d61375b8ab53be4c233f5b93e463edfdff9316a', 'targets': {'smithi135.front.sepia.ceph.com': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBJh1m5zJUpHrqEyHhqUVObJhAgM7iP05jGzbnT7mUeekySVDe8iX2BAY4xdaDHj9DekK4x3kUZUrLQKh3KfaYjY=', 'smithi164.front.sepia.ceph.com': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBNmOzx19qAeUG8w7yLVIdwoPWkyQAh2DMyIlga3wbcZH8cBrjHKdqq+xguBx2x/IgM0PT6Rzgxf2HcPAt7MdHzI='}, 'tasks': [{'internal.check_packages': None}, {'internal.buildpackages_prep': None}, {'internal.save_config': None}, {'internal.check_lock': None}, {'internal.add_remotes': None}, {'console_log': None}, {'internal.connect': None}, {'internal.push_inventory': None}, {'internal.serialize_remote_roles': None}, {'internal.check_conflict': None}, {'internal.check_ceph_data': None}, {'internal.vm_setup': None}, {'kernel': {'kdb': 1, 'sha1': 'distro'}}, {'internal.base': None}, {'internal.archive_upload': None}, {'internal.archive': None}, {'internal.coredump': None}, {'internal.sudo': None}, {'internal.syslog': None}, {'internal.timer': None}, {'pcp': None}, {'selinux': None}, {'ansible.cephlab': None}, {'clock': None}, {'install': {'extra_packages': ['rbd-mirror'], 'flavor': 'default', 'sha1': '7f24e589328ae9e639c0009d93340e5b235d0a47'}}, {'ceph': {'cluster': 'cluster1', 'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'simple'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bluestore block size': 96636764160, 'bluestore compression algorithm': 'zlib', 'bluestore compression mode': 'aggressive', 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': '7f24e589328ae9e639c0009d93340e5b235d0a47'}}, {'ceph': {'cluster': 'cluster2'}}, {'exec': {'cluster1.client.mirror.0': ["sudo ceph --cluster cluster1 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd'"]}}, {'rbd-mirror': {'client': 'cluster1.client.mirror.0', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster1.client.mirror.1', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster1.client.mirror.2', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster1.client.mirror.3', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster2.client.mirror.0', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster2.client.mirror.1', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster2.client.mirror.2', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster2.client.mirror.3', 'thrash': True}}, {'rbd-mirror-thrash': {'cluster': 'cluster1'}}, {'rbd-mirror-thrash': {'cluster': 'cluster2'}}, {'workunit': {'clients': {'cluster1.client.mirror': ['rbd/rbd_mirror_stress.sh']}, 'env': {'CEPH_ARGS': '', 'MIRROR_POOL_MODE': 'image', 'RBD_IMAGE_FEATURES': 'layering', 'RBD_MIRROR_INSTANCES': '4', 'RBD_MIRROR_MODE': 'snapshot', 'RBD_MIRROR_USE_EXISTING_CLUSTER': '1', 'RBD_MIRROR_USE_RBD_MIRROR': '1'}, 'timeout': '6h'}}], 'teuthology': {'fragments_dropped': [], 'meta': {}, 'postmerge': []}, 'teuthology_branch': 'main', 'teuthology_sha1': '3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a', 'timestamp': '2024-09-10_20:16:17', 'tube': 'smithi', 'user': 'teuthology', 'verbose': False, 'worker_log': '/home/teuthworker/archive/worker_logs/dispatcher.smithi.3401856'} 2024-09-14T08:32:19.967 DEBUG:teuthology.orchestra.run.smithi135:> install -d -m0755 -- /home/ubuntu/cephtest/cluster1.data 2024-09-14T08:32:19.993 DEBUG:teuthology.orchestra.run.smithi164:> install -d -m0755 -- /home/ubuntu/cephtest/cluster1.data 2024-09-14T08:32:20.009 DEBUG:teuthology.orchestra.run.smithi135:> sudo install -d -m0777 -- /var/run/ceph 2024-09-14T08:32:20.049 DEBUG:teuthology.orchestra.run.smithi164:> sudo install -d -m0777 -- /var/run/ceph 2024-09-14T08:32:20.074 DEBUG:teuthology.orchestra.run.smithi135:> set -ex 2024-09-14T08:32:20.074 DEBUG:teuthology.orchestra.run.smithi135:> dd if=/scratch_devs of=/dev/stdout 2024-09-14T08:32:20.127 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-09-14T08:32:20.128 DEBUG:teuthology.orchestra.run.smithi135:> stat /dev/vg_nvme/lv_1 2024-09-14T08:32:20.182 INFO:teuthology.orchestra.run.smithi135.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-09-14T08:32:20.182 INFO:teuthology.orchestra.run.smithi135.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-14T08:32:20.182 INFO:teuthology.orchestra.run.smithi135.stdout:Device: 5h/5d Inode: 667 Links: 1 2024-09-14T08:32:20.183 INFO:teuthology.orchestra.run.smithi135.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-14T08:32:20.183 INFO:teuthology.orchestra.run.smithi135.stdout:Context: system_u:object_r:device_t:s0 2024-09-14T08:32:20.183 INFO:teuthology.orchestra.run.smithi135.stdout:Access: 2024-09-14 08:32:09.145129272 +0000 2024-09-14T08:32:20.183 INFO:teuthology.orchestra.run.smithi135.stdout:Modify: 2024-09-14 08:29:38.445067710 +0000 2024-09-14T08:32:20.183 INFO:teuthology.orchestra.run.smithi135.stdout:Change: 2024-09-14 08:29:38.445067710 +0000 2024-09-14T08:32:20.183 INFO:teuthology.orchestra.run.smithi135.stdout: Birth: 2024-09-14 08:29:38.445067710 +0000 2024-09-14T08:32:20.183 DEBUG:teuthology.orchestra.run.smithi135:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-09-14T08:32:20.245 INFO:teuthology.orchestra.run.smithi135.stderr:1+0 records in 2024-09-14T08:32:20.245 INFO:teuthology.orchestra.run.smithi135.stderr:1+0 records out 2024-09-14T08:32:20.245 INFO:teuthology.orchestra.run.smithi135.stderr:512 bytes copied, 0.000266044 s, 1.9 MB/s 2024-09-14T08:32:20.246 DEBUG:teuthology.orchestra.run.smithi135:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-09-14T08:32:20.302 DEBUG:teuthology.orchestra.run.smithi135:> stat /dev/vg_nvme/lv_2 2024-09-14T08:32:20.357 INFO:teuthology.orchestra.run.smithi135.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-09-14T08:32:20.358 INFO:teuthology.orchestra.run.smithi135.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-14T08:32:20.358 INFO:teuthology.orchestra.run.smithi135.stdout:Device: 5h/5d Inode: 686 Links: 1 2024-09-14T08:32:20.358 INFO:teuthology.orchestra.run.smithi135.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-14T08:32:20.358 INFO:teuthology.orchestra.run.smithi135.stdout:Context: system_u:object_r:device_t:s0 2024-09-14T08:32:20.358 INFO:teuthology.orchestra.run.smithi135.stdout:Access: 2024-09-14 08:32:09.146129251 +0000 2024-09-14T08:32:20.358 INFO:teuthology.orchestra.run.smithi135.stdout:Modify: 2024-09-14 08:29:38.763061278 +0000 2024-09-14T08:32:20.358 INFO:teuthology.orchestra.run.smithi135.stdout:Change: 2024-09-14 08:29:38.763061278 +0000 2024-09-14T08:32:20.358 INFO:teuthology.orchestra.run.smithi135.stdout: Birth: 2024-09-14 08:29:38.763061278 +0000 2024-09-14T08:32:20.358 DEBUG:teuthology.orchestra.run.smithi135:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-09-14T08:32:20.420 INFO:teuthology.orchestra.run.smithi135.stderr:1+0 records in 2024-09-14T08:32:20.421 INFO:teuthology.orchestra.run.smithi135.stderr:1+0 records out 2024-09-14T08:32:20.421 INFO:teuthology.orchestra.run.smithi135.stderr:512 bytes copied, 0.000210412 s, 2.4 MB/s 2024-09-14T08:32:20.422 DEBUG:teuthology.orchestra.run.smithi135:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-09-14T08:32:20.478 DEBUG:teuthology.orchestra.run.smithi135:> stat /dev/vg_nvme/lv_3 2024-09-14T08:32:20.533 INFO:teuthology.orchestra.run.smithi135.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-09-14T08:32:20.533 INFO:teuthology.orchestra.run.smithi135.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-14T08:32:20.534 INFO:teuthology.orchestra.run.smithi135.stdout:Device: 5h/5d Inode: 703 Links: 1 2024-09-14T08:32:20.534 INFO:teuthology.orchestra.run.smithi135.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-14T08:32:20.534 INFO:teuthology.orchestra.run.smithi135.stdout:Context: system_u:object_r:device_t:s0 2024-09-14T08:32:20.534 INFO:teuthology.orchestra.run.smithi135.stdout:Access: 2024-09-14 08:32:09.146129251 +0000 2024-09-14T08:32:20.534 INFO:teuthology.orchestra.run.smithi135.stdout:Modify: 2024-09-14 08:29:39.097054523 +0000 2024-09-14T08:32:20.534 INFO:teuthology.orchestra.run.smithi135.stdout:Change: 2024-09-14 08:29:39.097054523 +0000 2024-09-14T08:32:20.534 INFO:teuthology.orchestra.run.smithi135.stdout: Birth: 2024-09-14 08:29:39.097054523 +0000 2024-09-14T08:32:20.534 DEBUG:teuthology.orchestra.run.smithi135:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-09-14T08:32:20.596 INFO:teuthology.orchestra.run.smithi135.stderr:1+0 records in 2024-09-14T08:32:20.596 INFO:teuthology.orchestra.run.smithi135.stderr:1+0 records out 2024-09-14T08:32:20.596 INFO:teuthology.orchestra.run.smithi135.stderr:512 bytes copied, 0.000190015 s, 2.7 MB/s 2024-09-14T08:32:20.598 DEBUG:teuthology.orchestra.run.smithi135:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-09-14T08:32:20.654 DEBUG:teuthology.orchestra.run.smithi135:> stat /dev/vg_nvme/lv_4 2024-09-14T08:32:20.710 INFO:teuthology.orchestra.run.smithi135.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-09-14T08:32:20.710 INFO:teuthology.orchestra.run.smithi135.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-14T08:32:20.710 INFO:teuthology.orchestra.run.smithi135.stdout:Device: 5h/5d Inode: 722 Links: 1 2024-09-14T08:32:20.711 INFO:teuthology.orchestra.run.smithi135.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-14T08:32:20.711 INFO:teuthology.orchestra.run.smithi135.stdout:Context: system_u:object_r:device_t:s0 2024-09-14T08:32:20.711 INFO:teuthology.orchestra.run.smithi135.stdout:Access: 2024-09-14 08:32:09.147129231 +0000 2024-09-14T08:32:20.711 INFO:teuthology.orchestra.run.smithi135.stdout:Modify: 2024-09-14 08:29:39.405048294 +0000 2024-09-14T08:32:20.711 INFO:teuthology.orchestra.run.smithi135.stdout:Change: 2024-09-14 08:29:39.405048294 +0000 2024-09-14T08:32:20.711 INFO:teuthology.orchestra.run.smithi135.stdout: Birth: 2024-09-14 08:29:39.405048294 +0000 2024-09-14T08:32:20.711 DEBUG:teuthology.orchestra.run.smithi135:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-09-14T08:32:20.773 INFO:teuthology.orchestra.run.smithi135.stderr:1+0 records in 2024-09-14T08:32:20.773 INFO:teuthology.orchestra.run.smithi135.stderr:1+0 records out 2024-09-14T08:32:20.773 INFO:teuthology.orchestra.run.smithi135.stderr:512 bytes copied, 0.000211671 s, 2.4 MB/s 2024-09-14T08:32:20.775 DEBUG:teuthology.orchestra.run.smithi135:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-09-14T08:32:20.831 INFO:tasks.ceph:osd dev map: {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'} 2024-09-14T08:32:20.831 INFO:tasks.ceph:remote_to_roles_to_devs: {Remote(name='ubuntu@smithi135.front.sepia.ceph.com'): {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'}} 2024-09-14T08:32:20.831 INFO:tasks.ceph:Generating config... 2024-09-14T08:32:20.833 INFO:tasks.ceph:[client] debug rbd = 20 2024-09-14T08:32:20.833 INFO:tasks.ceph:[client] debug rbd_mirror = 15 2024-09-14T08:32:20.833 INFO:tasks.ceph:[client] log to stderr = False 2024-09-14T08:32:20.833 INFO:tasks.ceph:[client] rbd default features = 125 2024-09-14T08:32:20.833 INFO:tasks.ceph:[client] rbd mirror image policy type = simple 2024-09-14T08:32:20.833 INFO:tasks.ceph:[client.mirror.0] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-14T08:32:20.833 INFO:tasks.ceph:[client.mirror.0] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-14T08:32:20.833 INFO:tasks.ceph:[client.mirror.1] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-14T08:32:20.833 INFO:tasks.ceph:[client.mirror.1] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-14T08:32:20.834 INFO:tasks.ceph:[client.mirror.2] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-14T08:32:20.834 INFO:tasks.ceph:[client.mirror.2] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-14T08:32:20.834 INFO:tasks.ceph:[client.mirror.3] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-14T08:32:20.834 INFO:tasks.ceph:[client.mirror.3] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-14T08:32:20.834 INFO:tasks.ceph:[global] mon client directed command retry = 5 2024-09-14T08:32:20.834 INFO:tasks.ceph:[global] mon warn on pool no app = False 2024-09-14T08:32:20.834 INFO:tasks.ceph:[global] ms inject socket failures = 5000 2024-09-14T08:32:20.834 INFO:tasks.ceph:[mgr] debug mgr = 20 2024-09-14T08:32:20.834 INFO:tasks.ceph:[mgr] debug ms = 1 2024-09-14T08:32:20.834 INFO:tasks.ceph:[mon] debug mon = 20 2024-09-14T08:32:20.834 INFO:tasks.ceph:[mon] debug ms = 1 2024-09-14T08:32:20.834 INFO:tasks.ceph:[mon] debug paxos = 20 2024-09-14T08:32:20.834 INFO:tasks.ceph:[osd] bluestore block size = 96636764160 2024-09-14T08:32:20.835 INFO:tasks.ceph:[osd] bluestore compression algorithm = zlib 2024-09-14T08:32:20.835 INFO:tasks.ceph:[osd] bluestore compression mode = aggressive 2024-09-14T08:32:20.835 INFO:tasks.ceph:[osd] bluestore fsck on mount = True 2024-09-14T08:32:20.835 INFO:tasks.ceph:[osd] debug bluefs = 1/20 2024-09-14T08:32:20.835 INFO:tasks.ceph:[osd] debug bluestore = 1/20 2024-09-14T08:32:20.835 INFO:tasks.ceph:[osd] debug ms = 1 2024-09-14T08:32:20.835 INFO:tasks.ceph:[osd] debug osd = 20 2024-09-14T08:32:20.835 INFO:tasks.ceph:[osd] debug rocksdb = 4/10 2024-09-14T08:32:20.835 INFO:tasks.ceph:[osd] mon osd backfillfull_ratio = 0.85 2024-09-14T08:32:20.835 INFO:tasks.ceph:[osd] mon osd full ratio = 0.9 2024-09-14T08:32:20.835 INFO:tasks.ceph:[osd] mon osd nearfull ratio = 0.8 2024-09-14T08:32:20.835 INFO:tasks.ceph:[osd] osd failsafe full ratio = 0.95 2024-09-14T08:32:20.835 INFO:tasks.ceph:[osd] osd objectstore = bluestore 2024-09-14T08:32:20.836 INFO:tasks.ceph:Setting up cluster1.mon.a... 2024-09-14T08:32:20.836 DEBUG:teuthology.orchestra.run.smithi135:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring /etc/ceph/cluster1.keyring 2024-09-14T08:32:20.923 INFO:teuthology.orchestra.run.smithi135.stdout:creating /etc/ceph/cluster1.keyring 2024-09-14T08:32:20.926 DEBUG:teuthology.orchestra.run.smithi135:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=mon. /etc/ceph/cluster1.keyring 2024-09-14T08:32:20.968 DEBUG:teuthology.orchestra.run.smithi135:> sudo chmod 0644 /etc/ceph/cluster1.keyring 2024-09-14T08:32:21.033 DEBUG:tasks.ceph:Ceph mon addresses: [('cluster1.mon.a', '172.21.15.135')] 2024-09-14T08:32:21.033 DEBUG:tasks.ceph:writing out conf {'global': {'chdir': '', 'pid file': '/var/run/ceph/$cluster-$name.pid', 'auth supported': 'cephx', 'filestore xattr use omap': 'true', 'mon clock drift allowed': '1.000', 'osd crush chooseleaf type': '0', 'auth debug': 'true', 'ms die on old message': 'true', 'ms die on bug': 'true', 'mon max pg per osd': '10000', 'mon pg warn max object skew': '0', 'osd_pool_default_pg_autoscale_mode': 'off', 'osd pool default size': '2', 'mon osd allow primary affinity': 'true', 'mon osd allow pg remap': 'true', 'mon warn on legacy crush tunables': 'false', 'mon warn on crush straw calc version zero': 'false', 'mon warn on no sortbitwise': 'false', 'mon warn on osd down out interval zero': 'false', 'mon warn on too few osds': 'false', 'mon_warn_on_pool_pg_num_not_power_of_two': 'false', 'mon_warn_on_pool_no_redundancy': 'false', 'mon_allow_pool_size_one': 'true', 'osd pool default erasure code profile': 'plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd', 'osd default data pool replay window': '5', 'mon allow pool delete': 'true', 'mon cluster log file level': 'debug', 'debug asserts on shutdown': 'true', 'mon health detail to clog': 'false', 'mon host': '172.21.15.135', 'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'osd': {'osd journal size': '100', 'osd scrub load threshold': '5.0', 'osd scrub max interval': '600', 'osd mclock profile': 'high_recovery_ops', 'osd recover clone overlap': 'true', 'osd recovery max chunk': '1048576', 'osd debug shutdown': 'true', 'osd debug op order': 'true', 'osd debug verify stray on activate': 'true', 'osd debug trim objects': 'true', 'osd open classes on start': 'true', 'osd debug pg log writeout': 'true', 'osd deep scrub update digest min age': '30', 'osd map max advance': '10', 'journal zero on create': 'true', 'filestore ondisk finisher threads': '3', 'filestore apply finisher threads': '3', 'bdev debug aio': 'true', 'osd debug misdirected ops': 'true', 'bluestore block size': 96636764160, 'bluestore compression algorithm': 'zlib', 'bluestore compression mode': 'aggressive', 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}, 'mgr': {'debug ms': 1, 'debug mgr': 20, 'debug mon': '20', 'debug auth': '20', 'mon reweight min pgs per osd': '4', 'mon reweight min bytes per osd': '10', 'mgr/telemetry/nag': 'false'}, 'mon': {'debug ms': 1, 'debug mon': 20, 'debug paxos': 20, 'debug auth': '20', 'mon data avail warn': '5', 'mon mgr mkfs grace': '240', 'mon reweight min pgs per osd': '4', 'mon osd reporter subtree level': 'osd', 'mon osd prime pg temp': 'true', 'mon reweight min bytes per osd': '10', 'auth mon ticket ttl': '660', 'auth service ticket ttl': '240', 'mon_warn_on_insecure_global_id_reclaim': 'false', 'mon_warn_on_insecure_global_id_reclaim_allowed': 'false', 'mon_down_mkfs_grace': '2m', 'mon_warn_on_filestore_osds': 'false'}, 'client': {'rgw cache enabled': 'true', 'rgw enable ops log': 'true', 'rgw enable usage log': 'true', 'log file': '/var/log/ceph/$cluster-$name.$pid.log', 'admin socket': '/var/run/ceph/$cluster-$name.$pid.asok', 'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'simple'}, 'mon.a': {}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}} 2024-09-14T08:32:21.034 DEBUG:teuthology.orchestra.run.smithi135:> set -ex 2024-09-14T08:32:21.034 DEBUG:teuthology.orchestra.run.smithi135:> dd of=/home/ubuntu/cephtest/ceph.tmp.conf 2024-09-14T08:32:21.089 DEBUG:teuthology.orchestra.run.smithi135:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage monmaptool -c /home/ubuntu/cephtest/ceph.tmp.conf --create --clobber --enable-all-features --add a 172.21.15.135 --print /home/ubuntu/cephtest/cluster1.monmap 2024-09-14T08:32:21.169 INFO:teuthology.orchestra.run.smithi135.stdout:monmaptool: monmap file /home/ubuntu/cephtest/cluster1.monmap 2024-09-14T08:32:21.169 INFO:teuthology.orchestra.run.smithi135.stdout:monmaptool: generated fsid 77c3c5ea-bdba-488a-b29d-8702bc143d62 2024-09-14T08:32:21.169 INFO:teuthology.orchestra.run.smithi135.stdout:setting min_mon_release = quincy 2024-09-14T08:32:21.169 INFO:teuthology.orchestra.run.smithi135.stdout:epoch 0 2024-09-14T08:32:21.170 INFO:teuthology.orchestra.run.smithi135.stdout:fsid 77c3c5ea-bdba-488a-b29d-8702bc143d62 2024-09-14T08:32:21.170 INFO:teuthology.orchestra.run.smithi135.stdout:last_changed 2024-09-14T08:32:21.170889+0000 2024-09-14T08:32:21.170 INFO:teuthology.orchestra.run.smithi135.stdout:created 2024-09-14T08:32:21.170889+0000 2024-09-14T08:32:21.170 INFO:teuthology.orchestra.run.smithi135.stdout:min_mon_release 17 (quincy) 2024-09-14T08:32:21.170 INFO:teuthology.orchestra.run.smithi135.stdout:election_strategy: 1 2024-09-14T08:32:21.170 INFO:teuthology.orchestra.run.smithi135.stdout:0: [v2:172.21.15.135:3300/0,v1:172.21.15.135:6789/0] mon.a 2024-09-14T08:32:21.170 INFO:teuthology.orchestra.run.smithi135.stdout:monmaptool: writing epoch 0 to /home/ubuntu/cephtest/cluster1.monmap (1 monitors) 2024-09-14T08:32:21.171 DEBUG:teuthology.orchestra.run.smithi135:> rm -- /home/ubuntu/cephtest/ceph.tmp.conf 2024-09-14T08:32:21.226 INFO:tasks.ceph:Writing /etc/ceph/cluster1.conf for FSID 77c3c5ea-bdba-488a-b29d-8702bc143d62... 2024-09-14T08:32:21.227 DEBUG:teuthology.orchestra.run.smithi135:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/cluster1.conf && sudo chmod 0644 /etc/ceph/cluster1.conf > /dev/null 2024-09-14T08:32:21.269 DEBUG:teuthology.orchestra.run.smithi164:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/cluster1.conf && sudo chmod 0644 /etc/ceph/cluster1.conf > /dev/null 2024-09-14T08:32:21.309 INFO:teuthology.orchestra.run.smithi135.stdout:[global] 2024-09-14T08:32:21.309 INFO:teuthology.orchestra.run.smithi135.stdout: chdir = "" 2024-09-14T08:32:21.309 INFO:teuthology.orchestra.run.smithi135.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2024-09-14T08:32:21.309 INFO:teuthology.orchestra.run.smithi135.stdout: auth supported = cephx 2024-09-14T08:32:21.309 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:32:21.309 INFO:teuthology.orchestra.run.smithi135.stdout: filestore xattr use omap = true 2024-09-14T08:32:21.309 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:32:21.309 INFO:teuthology.orchestra.run.smithi135.stdout: mon clock drift allowed = 1.000 2024-09-14T08:32:21.310 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:32:21.310 INFO:teuthology.orchestra.run.smithi135.stdout: osd crush chooseleaf type = 0 2024-09-14T08:32:21.310 INFO:teuthology.orchestra.run.smithi135.stdout: auth debug = true 2024-09-14T08:32:21.310 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:32:21.310 INFO:teuthology.orchestra.run.smithi135.stdout: ms die on old message = true 2024-09-14T08:32:21.310 INFO:teuthology.orchestra.run.smithi135.stdout: ms die on bug = true 2024-09-14T08:32:21.310 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:32:21.310 INFO:teuthology.orchestra.run.smithi135.stdout: mon max pg per osd = 10000 # >= luminous 2024-09-14T08:32:21.310 INFO:teuthology.orchestra.run.smithi135.stdout: mon pg warn max object skew = 0 2024-09-14T08:32:21.310 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:32:21.310 INFO:teuthology.orchestra.run.smithi135.stdout: # disable pg_autoscaler by default for new pools 2024-09-14T08:32:21.310 INFO:teuthology.orchestra.run.smithi135.stdout: osd_pool_default_pg_autoscale_mode = off 2024-09-14T08:32:21.310 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:32:21.311 INFO:teuthology.orchestra.run.smithi135.stdout: osd pool default size = 2 2024-09-14T08:32:21.311 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:32:21.311 INFO:teuthology.orchestra.run.smithi135.stdout: mon osd allow primary affinity = true 2024-09-14T08:32:21.312 INFO:teuthology.orchestra.run.smithi135.stdout: mon osd allow pg remap = true 2024-09-14T08:32:21.312 INFO:teuthology.orchestra.run.smithi135.stdout: mon warn on legacy crush tunables = false 2024-09-14T08:32:21.312 INFO:teuthology.orchestra.run.smithi135.stdout: mon warn on crush straw calc version zero = false 2024-09-14T08:32:21.312 INFO:teuthology.orchestra.run.smithi135.stdout: mon warn on no sortbitwise = false 2024-09-14T08:32:21.312 INFO:teuthology.orchestra.run.smithi135.stdout: mon warn on osd down out interval zero = false 2024-09-14T08:32:21.312 INFO:teuthology.orchestra.run.smithi135.stdout: mon warn on too few osds = false 2024-09-14T08:32:21.312 INFO:teuthology.orchestra.run.smithi135.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2024-09-14T08:32:21.312 INFO:teuthology.orchestra.run.smithi135.stdout: mon_warn_on_pool_no_redundancy = false 2024-09-14T08:32:21.312 INFO:teuthology.orchestra.run.smithi135.stdout: mon_allow_pool_size_one = true 2024-09-14T08:32:21.312 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:32:21.312 INFO:teuthology.orchestra.run.smithi135.stdout: osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2024-09-14T08:32:21.312 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:32:21.312 INFO:teuthology.orchestra.run.smithi135.stdout: osd default data pool replay window = 5 2024-09-14T08:32:21.313 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:32:21.313 INFO:teuthology.orchestra.run.smithi135.stdout: mon allow pool delete = true 2024-09-14T08:32:21.313 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:32:21.313 INFO:teuthology.orchestra.run.smithi135.stdout: mon cluster log file level = debug 2024-09-14T08:32:21.313 INFO:teuthology.orchestra.run.smithi135.stdout: debug asserts on shutdown = true 2024-09-14T08:32:21.313 INFO:teuthology.orchestra.run.smithi135.stdout: mon health detail to clog = false 2024-09-14T08:32:21.313 INFO:teuthology.orchestra.run.smithi135.stdout: mon host = 172.21.15.135 2024-09-14T08:32:21.313 INFO:teuthology.orchestra.run.smithi135.stdout: mon client directed command retry = 5 2024-09-14T08:32:21.313 INFO:teuthology.orchestra.run.smithi135.stdout: mon warn on pool no app = False 2024-09-14T08:32:21.313 INFO:teuthology.orchestra.run.smithi135.stdout: ms inject socket failures = 5000 2024-09-14T08:32:21.313 INFO:teuthology.orchestra.run.smithi135.stdout: fsid = 77c3c5ea-bdba-488a-b29d-8702bc143d62 2024-09-14T08:32:21.313 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:32:21.313 INFO:teuthology.orchestra.run.smithi135.stdout:[osd] 2024-09-14T08:32:21.313 INFO:teuthology.orchestra.run.smithi135.stdout: osd journal size = 100 2024-09-14T08:32:21.314 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:32:21.314 INFO:teuthology.orchestra.run.smithi135.stdout: osd scrub load threshold = 5.0 2024-09-14T08:32:21.314 INFO:teuthology.orchestra.run.smithi135.stdout: osd scrub max interval = 600 2024-09-14T08:32:21.314 INFO:teuthology.orchestra.run.smithi135.stdout: osd mclock profile = high_recovery_ops 2024-09-14T08:32:21.314 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:32:21.314 INFO:teuthology.orchestra.run.smithi135.stdout: osd recover clone overlap = true 2024-09-14T08:32:21.314 INFO:teuthology.orchestra.run.smithi135.stdout: osd recovery max chunk = 1048576 2024-09-14T08:32:21.314 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:32:21.314 INFO:teuthology.orchestra.run.smithi135.stdout: osd debug shutdown = true 2024-09-14T08:32:21.314 INFO:teuthology.orchestra.run.smithi135.stdout: osd debug op order = true 2024-09-14T08:32:21.314 INFO:teuthology.orchestra.run.smithi135.stdout: osd debug verify stray on activate = true 2024-09-14T08:32:21.314 INFO:teuthology.orchestra.run.smithi135.stdout: osd debug trim objects = true 2024-09-14T08:32:21.314 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:32:21.314 INFO:teuthology.orchestra.run.smithi135.stdout: osd open classes on start = true 2024-09-14T08:32:21.314 INFO:teuthology.orchestra.run.smithi135.stdout: osd debug pg log writeout = true 2024-09-14T08:32:21.315 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:32:21.315 INFO:teuthology.orchestra.run.smithi135.stdout: osd deep scrub update digest min age = 30 2024-09-14T08:32:21.315 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:32:21.315 INFO:teuthology.orchestra.run.smithi135.stdout: osd map max advance = 10 2024-09-14T08:32:21.315 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:32:21.315 INFO:teuthology.orchestra.run.smithi135.stdout: journal zero on create = true 2024-09-14T08:32:21.315 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:32:21.315 INFO:teuthology.orchestra.run.smithi135.stdout: filestore ondisk finisher threads = 3 2024-09-14T08:32:21.315 INFO:teuthology.orchestra.run.smithi135.stdout: filestore apply finisher threads = 3 2024-09-14T08:32:21.315 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:32:21.315 INFO:teuthology.orchestra.run.smithi135.stdout: bdev debug aio = true 2024-09-14T08:32:21.315 INFO:teuthology.orchestra.run.smithi135.stdout: osd debug misdirected ops = true 2024-09-14T08:32:21.315 INFO:teuthology.orchestra.run.smithi135.stdout: bluestore block size = 96636764160 2024-09-14T08:32:21.316 INFO:teuthology.orchestra.run.smithi135.stdout: bluestore compression algorithm = zlib 2024-09-14T08:32:21.316 INFO:teuthology.orchestra.run.smithi135.stdout: bluestore compression mode = aggressive 2024-09-14T08:32:21.316 INFO:teuthology.orchestra.run.smithi135.stdout: bluestore fsck on mount = True 2024-09-14T08:32:21.316 INFO:teuthology.orchestra.run.smithi135.stdout: debug bluefs = 1/20 2024-09-14T08:32:21.316 INFO:teuthology.orchestra.run.smithi135.stdout: debug bluestore = 1/20 2024-09-14T08:32:21.316 INFO:teuthology.orchestra.run.smithi135.stdout: debug ms = 1 2024-09-14T08:32:21.316 INFO:teuthology.orchestra.run.smithi135.stdout: debug osd = 20 2024-09-14T08:32:21.316 INFO:teuthology.orchestra.run.smithi135.stdout: debug rocksdb = 4/10 2024-09-14T08:32:21.316 INFO:teuthology.orchestra.run.smithi135.stdout: mon osd backfillfull_ratio = 0.85 2024-09-14T08:32:21.316 INFO:teuthology.orchestra.run.smithi135.stdout: mon osd full ratio = 0.9 2024-09-14T08:32:21.316 INFO:teuthology.orchestra.run.smithi135.stdout: mon osd nearfull ratio = 0.8 2024-09-14T08:32:21.317 INFO:teuthology.orchestra.run.smithi135.stdout: osd failsafe full ratio = 0.95 2024-09-14T08:32:21.317 INFO:teuthology.orchestra.run.smithi135.stdout: osd objectstore = bluestore 2024-09-14T08:32:21.317 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:32:21.317 INFO:teuthology.orchestra.run.smithi135.stdout:[mgr] 2024-09-14T08:32:21.317 INFO:teuthology.orchestra.run.smithi135.stdout: debug ms = 1 2024-09-14T08:32:21.317 INFO:teuthology.orchestra.run.smithi135.stdout: debug mgr = 20 2024-09-14T08:32:21.317 INFO:teuthology.orchestra.run.smithi135.stdout: debug mon = 20 2024-09-14T08:32:21.317 INFO:teuthology.orchestra.run.smithi135.stdout: debug auth = 20 2024-09-14T08:32:21.317 INFO:teuthology.orchestra.run.smithi135.stdout: mon reweight min pgs per osd = 4 2024-09-14T08:32:21.317 INFO:teuthology.orchestra.run.smithi135.stdout: mon reweight min bytes per osd = 10 2024-09-14T08:32:21.317 INFO:teuthology.orchestra.run.smithi135.stdout: mgr/telemetry/nag = false 2024-09-14T08:32:21.317 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:32:21.317 INFO:teuthology.orchestra.run.smithi135.stdout:[mon] 2024-09-14T08:32:21.317 INFO:teuthology.orchestra.run.smithi135.stdout: debug ms = 1 2024-09-14T08:32:21.318 INFO:teuthology.orchestra.run.smithi135.stdout: debug mon = 20 2024-09-14T08:32:21.318 INFO:teuthology.orchestra.run.smithi135.stdout: debug paxos = 20 2024-09-14T08:32:21.318 INFO:teuthology.orchestra.run.smithi135.stdout: debug auth = 20 2024-09-14T08:32:21.318 INFO:teuthology.orchestra.run.smithi135.stdout: mon data avail warn = 5 2024-09-14T08:32:21.318 INFO:teuthology.orchestra.run.smithi135.stdout: mon mgr mkfs grace = 240 2024-09-14T08:32:21.318 INFO:teuthology.orchestra.run.smithi135.stdout: mon reweight min pgs per osd = 4 2024-09-14T08:32:21.318 INFO:teuthology.orchestra.run.smithi135.stdout: mon osd reporter subtree level = osd 2024-09-14T08:32:21.318 INFO:teuthology.orchestra.run.smithi135.stdout: mon osd prime pg temp = true 2024-09-14T08:32:21.318 INFO:teuthology.orchestra.run.smithi135.stdout: mon reweight min bytes per osd = 10 2024-09-14T08:32:21.318 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:32:21.318 INFO:teuthology.orchestra.run.smithi135.stdout: # rotate auth tickets quickly to exercise renewal paths 2024-09-14T08:32:21.318 INFO:teuthology.orchestra.run.smithi135.stdout: auth mon ticket ttl = 660 # 11m 2024-09-14T08:32:21.318 INFO:teuthology.orchestra.run.smithi135.stdout: auth service ticket ttl = 240 # 4m 2024-09-14T08:32:21.319 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:32:21.319 INFO:teuthology.orchestra.run.smithi135.stdout: # don't complain about insecure global_id in the test suite 2024-09-14T08:32:21.319 INFO:teuthology.orchestra.run.smithi135.stdout: mon_warn_on_insecure_global_id_reclaim = false 2024-09-14T08:32:21.319 INFO:teuthology.orchestra.run.smithi135.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2024-09-14T08:32:21.319 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:32:21.319 INFO:teuthology.orchestra.run.smithi135.stdout: # 1m isn't quite enough 2024-09-14T08:32:21.319 INFO:teuthology.orchestra.run.smithi135.stdout: mon_down_mkfs_grace = 2m 2024-09-14T08:32:21.319 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:32:21.319 INFO:teuthology.orchestra.run.smithi135.stdout: mon_warn_on_filestore_osds = false 2024-09-14T08:32:21.319 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:32:21.319 INFO:teuthology.orchestra.run.smithi135.stdout:[client] 2024-09-14T08:32:21.320 INFO:teuthology.orchestra.run.smithi135.stdout: rgw cache enabled = true 2024-09-14T08:32:21.320 INFO:teuthology.orchestra.run.smithi135.stdout: rgw enable ops log = true 2024-09-14T08:32:21.320 INFO:teuthology.orchestra.run.smithi135.stdout: rgw enable usage log = true 2024-09-14T08:32:21.320 INFO:teuthology.orchestra.run.smithi135.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2024-09-14T08:32:21.320 INFO:teuthology.orchestra.run.smithi135.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2024-09-14T08:32:21.320 INFO:teuthology.orchestra.run.smithi135.stdout: debug rbd = 20 2024-09-14T08:32:21.320 INFO:teuthology.orchestra.run.smithi135.stdout: debug rbd_mirror = 15 2024-09-14T08:32:21.320 INFO:teuthology.orchestra.run.smithi135.stdout: log to stderr = False 2024-09-14T08:32:21.320 INFO:teuthology.orchestra.run.smithi135.stdout: rbd default features = 125 2024-09-14T08:32:21.320 INFO:teuthology.orchestra.run.smithi135.stdout: rbd mirror image policy type = simple 2024-09-14T08:32:21.320 INFO:teuthology.orchestra.run.smithi135.stdout:[mon.a] 2024-09-14T08:32:21.321 INFO:teuthology.orchestra.run.smithi135.stdout:[client.mirror.0] 2024-09-14T08:32:21.321 INFO:teuthology.orchestra.run.smithi135.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-14T08:32:21.321 INFO:teuthology.orchestra.run.smithi135.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-14T08:32:21.321 INFO:teuthology.orchestra.run.smithi135.stdout:[client.mirror.1] 2024-09-14T08:32:21.321 INFO:teuthology.orchestra.run.smithi135.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-14T08:32:21.321 INFO:teuthology.orchestra.run.smithi135.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-14T08:32:21.321 INFO:teuthology.orchestra.run.smithi135.stdout:[client.mirror.2] 2024-09-14T08:32:21.321 INFO:teuthology.orchestra.run.smithi135.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-14T08:32:21.321 INFO:teuthology.orchestra.run.smithi135.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-14T08:32:21.321 INFO:teuthology.orchestra.run.smithi135.stdout:[client.mirror.3] 2024-09-14T08:32:21.321 INFO:teuthology.orchestra.run.smithi135.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-14T08:32:21.321 INFO:teuthology.orchestra.run.smithi135.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-14T08:32:21.322 INFO:teuthology.orchestra.run.smithi164.stdout:[global] 2024-09-14T08:32:21.322 INFO:teuthology.orchestra.run.smithi164.stdout: chdir = "" 2024-09-14T08:32:21.322 INFO:teuthology.orchestra.run.smithi164.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2024-09-14T08:32:21.322 INFO:teuthology.orchestra.run.smithi164.stdout: auth supported = cephx 2024-09-14T08:32:21.322 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:32:21.323 INFO:teuthology.orchestra.run.smithi164.stdout: filestore xattr use omap = true 2024-09-14T08:32:21.323 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:32:21.323 INFO:teuthology.orchestra.run.smithi164.stdout: mon clock drift allowed = 1.000 2024-09-14T08:32:21.323 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:32:21.323 INFO:teuthology.orchestra.run.smithi164.stdout: osd crush chooseleaf type = 0 2024-09-14T08:32:21.323 INFO:teuthology.orchestra.run.smithi164.stdout: auth debug = true 2024-09-14T08:32:21.323 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:32:21.323 INFO:teuthology.orchestra.run.smithi164.stdout: ms die on old message = true 2024-09-14T08:32:21.323 INFO:teuthology.orchestra.run.smithi164.stdout: ms die on bug = true 2024-09-14T08:32:21.323 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:32:21.323 INFO:teuthology.orchestra.run.smithi164.stdout: mon max pg per osd = 10000 # >= luminous 2024-09-14T08:32:21.323 INFO:teuthology.orchestra.run.smithi164.stdout: mon pg warn max object skew = 0 2024-09-14T08:32:21.323 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:32:21.323 INFO:teuthology.orchestra.run.smithi164.stdout: # disable pg_autoscaler by default for new pools 2024-09-14T08:32:21.324 INFO:teuthology.orchestra.run.smithi164.stdout: osd_pool_default_pg_autoscale_mode = off 2024-09-14T08:32:21.324 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:32:21.324 INFO:teuthology.orchestra.run.smithi164.stdout: osd pool default size = 2 2024-09-14T08:32:21.324 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:32:21.324 INFO:teuthology.orchestra.run.smithi164.stdout: mon osd allow primary affinity = true 2024-09-14T08:32:21.324 INFO:teuthology.orchestra.run.smithi164.stdout: mon osd allow pg remap = true 2024-09-14T08:32:21.324 INFO:teuthology.orchestra.run.smithi164.stdout: mon warn on legacy crush tunables = false 2024-09-14T08:32:21.324 INFO:teuthology.orchestra.run.smithi164.stdout: mon warn on crush straw calc version zero = false 2024-09-14T08:32:21.324 INFO:teuthology.orchestra.run.smithi164.stdout: mon warn on no sortbitwise = false 2024-09-14T08:32:21.324 INFO:teuthology.orchestra.run.smithi164.stdout: mon warn on osd down out interval zero = false 2024-09-14T08:32:21.324 INFO:teuthology.orchestra.run.smithi164.stdout: mon warn on too few osds = false 2024-09-14T08:32:21.324 INFO:teuthology.orchestra.run.smithi164.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2024-09-14T08:32:21.324 INFO:teuthology.orchestra.run.smithi164.stdout: mon_warn_on_pool_no_redundancy = false 2024-09-14T08:32:21.325 INFO:teuthology.orchestra.run.smithi164.stdout: mon_allow_pool_size_one = true 2024-09-14T08:32:21.325 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:32:21.325 INFO:teuthology.orchestra.run.smithi164.stdout: osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2024-09-14T08:32:21.325 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:32:21.325 INFO:teuthology.orchestra.run.smithi164.stdout: osd default data pool replay window = 5 2024-09-14T08:32:21.325 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:32:21.325 INFO:teuthology.orchestra.run.smithi164.stdout: mon allow pool delete = true 2024-09-14T08:32:21.325 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:32:21.325 INFO:teuthology.orchestra.run.smithi164.stdout: mon cluster log file level = debug 2024-09-14T08:32:21.325 INFO:teuthology.orchestra.run.smithi164.stdout: debug asserts on shutdown = true 2024-09-14T08:32:21.325 INFO:teuthology.orchestra.run.smithi164.stdout: mon health detail to clog = false 2024-09-14T08:32:21.325 INFO:teuthology.orchestra.run.smithi164.stdout: mon host = 172.21.15.135 2024-09-14T08:32:21.325 INFO:teuthology.orchestra.run.smithi164.stdout: mon client directed command retry = 5 2024-09-14T08:32:21.326 INFO:teuthology.orchestra.run.smithi164.stdout: mon warn on pool no app = False 2024-09-14T08:32:21.326 INFO:teuthology.orchestra.run.smithi164.stdout: ms inject socket failures = 5000 2024-09-14T08:32:21.326 INFO:teuthology.orchestra.run.smithi164.stdout: fsid = 77c3c5ea-bdba-488a-b29d-8702bc143d62 2024-09-14T08:32:21.326 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:32:21.326 INFO:teuthology.orchestra.run.smithi164.stdout:[osd] 2024-09-14T08:32:21.326 INFO:teuthology.orchestra.run.smithi164.stdout: osd journal size = 100 2024-09-14T08:32:21.326 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:32:21.326 INFO:teuthology.orchestra.run.smithi164.stdout: osd scrub load threshold = 5.0 2024-09-14T08:32:21.326 INFO:teuthology.orchestra.run.smithi164.stdout: osd scrub max interval = 600 2024-09-14T08:32:21.326 INFO:teuthology.orchestra.run.smithi164.stdout: osd mclock profile = high_recovery_ops 2024-09-14T08:32:21.326 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:32:21.326 INFO:teuthology.orchestra.run.smithi164.stdout: osd recover clone overlap = true 2024-09-14T08:32:21.327 INFO:teuthology.orchestra.run.smithi164.stdout: osd recovery max chunk = 1048576 2024-09-14T08:32:21.327 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:32:21.327 INFO:teuthology.orchestra.run.smithi164.stdout: osd debug shutdown = true 2024-09-14T08:32:21.327 INFO:teuthology.orchestra.run.smithi164.stdout: osd debug op order = true 2024-09-14T08:32:21.327 INFO:teuthology.orchestra.run.smithi164.stdout: osd debug verify stray on activate = true 2024-09-14T08:32:21.327 INFO:teuthology.orchestra.run.smithi164.stdout: osd debug trim objects = true 2024-09-14T08:32:21.327 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:32:21.327 INFO:teuthology.orchestra.run.smithi164.stdout: osd open classes on start = true 2024-09-14T08:32:21.327 INFO:teuthology.orchestra.run.smithi164.stdout: osd debug pg log writeout = true 2024-09-14T08:32:21.327 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:32:21.327 INFO:teuthology.orchestra.run.smithi164.stdout: osd deep scrub update digest min age = 30 2024-09-14T08:32:21.327 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:32:21.327 INFO:teuthology.orchestra.run.smithi164.stdout: osd map max advance = 10 2024-09-14T08:32:21.327 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:32:21.327 INFO:teuthology.orchestra.run.smithi164.stdout: journal zero on create = true 2024-09-14T08:32:21.327 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:32:21.328 INFO:teuthology.orchestra.run.smithi164.stdout: filestore ondisk finisher threads = 3 2024-09-14T08:32:21.328 INFO:teuthology.orchestra.run.smithi164.stdout: filestore apply finisher threads = 3 2024-09-14T08:32:21.328 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:32:21.328 INFO:teuthology.orchestra.run.smithi164.stdout: bdev debug aio = true 2024-09-14T08:32:21.328 INFO:teuthology.orchestra.run.smithi164.stdout: osd debug misdirected ops = true 2024-09-14T08:32:21.328 INFO:teuthology.orchestra.run.smithi164.stdout: bluestore block size = 96636764160 2024-09-14T08:32:21.328 INFO:teuthology.orchestra.run.smithi164.stdout: bluestore compression algorithm = zlib 2024-09-14T08:32:21.328 INFO:teuthology.orchestra.run.smithi164.stdout: bluestore compression mode = aggressive 2024-09-14T08:32:21.328 INFO:teuthology.orchestra.run.smithi164.stdout: bluestore fsck on mount = True 2024-09-14T08:32:21.328 INFO:teuthology.orchestra.run.smithi164.stdout: debug bluefs = 1/20 2024-09-14T08:32:21.328 INFO:teuthology.orchestra.run.smithi164.stdout: debug bluestore = 1/20 2024-09-14T08:32:21.328 INFO:teuthology.orchestra.run.smithi164.stdout: debug ms = 1 2024-09-14T08:32:21.328 INFO:teuthology.orchestra.run.smithi164.stdout: debug osd = 20 2024-09-14T08:32:21.328 INFO:teuthology.orchestra.run.smithi164.stdout: debug rocksdb = 4/10 2024-09-14T08:32:21.328 INFO:teuthology.orchestra.run.smithi164.stdout: mon osd backfillfull_ratio = 0.85 2024-09-14T08:32:21.328 INFO:teuthology.orchestra.run.smithi164.stdout: mon osd full ratio = 0.9 2024-09-14T08:32:21.329 INFO:teuthology.orchestra.run.smithi164.stdout: mon osd nearfull ratio = 0.8 2024-09-14T08:32:21.329 INFO:teuthology.orchestra.run.smithi164.stdout: osd failsafe full ratio = 0.95 2024-09-14T08:32:21.329 INFO:teuthology.orchestra.run.smithi164.stdout: osd objectstore = bluestore 2024-09-14T08:32:21.329 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:32:21.329 INFO:teuthology.orchestra.run.smithi164.stdout:[mgr] 2024-09-14T08:32:21.329 INFO:teuthology.orchestra.run.smithi164.stdout: debug ms = 1 2024-09-14T08:32:21.329 INFO:teuthology.orchestra.run.smithi164.stdout: debug mgr = 20 2024-09-14T08:32:21.329 INFO:teuthology.orchestra.run.smithi164.stdout: debug mon = 20 2024-09-14T08:32:21.329 INFO:teuthology.orchestra.run.smithi164.stdout: debug auth = 20 2024-09-14T08:32:21.329 INFO:teuthology.orchestra.run.smithi164.stdout: mon reweight min pgs per osd = 4 2024-09-14T08:32:21.329 INFO:teuthology.orchestra.run.smithi164.stdout: mon reweight min bytes per osd = 10 2024-09-14T08:32:21.329 INFO:teuthology.orchestra.run.smithi164.stdout: mgr/telemetry/nag = false 2024-09-14T08:32:21.329 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:32:21.329 INFO:teuthology.orchestra.run.smithi164.stdout:[mon] 2024-09-14T08:32:21.329 INFO:teuthology.orchestra.run.smithi164.stdout: debug ms = 1 2024-09-14T08:32:21.330 INFO:teuthology.orchestra.run.smithi164.stdout: debug mon = 20 2024-09-14T08:32:21.330 INFO:teuthology.orchestra.run.smithi164.stdout: debug paxos = 20 2024-09-14T08:32:21.330 INFO:teuthology.orchestra.run.smithi164.stdout: debug auth = 20 2024-09-14T08:32:21.330 INFO:teuthology.orchestra.run.smithi164.stdout: mon data avail warn = 5 2024-09-14T08:32:21.330 INFO:teuthology.orchestra.run.smithi164.stdout: mon mgr mkfs grace = 240 2024-09-14T08:32:21.330 INFO:teuthology.orchestra.run.smithi164.stdout: mon reweight min pgs per osd = 4 2024-09-14T08:32:21.330 INFO:teuthology.orchestra.run.smithi164.stdout: mon osd reporter subtree level = osd 2024-09-14T08:32:21.330 INFO:teuthology.orchestra.run.smithi164.stdout: mon osd prime pg temp = true 2024-09-14T08:32:21.330 INFO:teuthology.orchestra.run.smithi164.stdout: mon reweight min bytes per osd = 10 2024-09-14T08:32:21.330 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:32:21.330 INFO:teuthology.orchestra.run.smithi164.stdout: # rotate auth tickets quickly to exercise renewal paths 2024-09-14T08:32:21.330 INFO:teuthology.orchestra.run.smithi164.stdout: auth mon ticket ttl = 660 # 11m 2024-09-14T08:32:21.330 INFO:teuthology.orchestra.run.smithi164.stdout: auth service ticket ttl = 240 # 4m 2024-09-14T08:32:21.330 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:32:21.331 INFO:teuthology.orchestra.run.smithi164.stdout: # don't complain about insecure global_id in the test suite 2024-09-14T08:32:21.331 INFO:teuthology.orchestra.run.smithi164.stdout: mon_warn_on_insecure_global_id_reclaim = false 2024-09-14T08:32:21.331 INFO:teuthology.orchestra.run.smithi164.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2024-09-14T08:32:21.331 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:32:21.331 INFO:teuthology.orchestra.run.smithi164.stdout: # 1m isn't quite enough 2024-09-14T08:32:21.331 INFO:teuthology.orchestra.run.smithi164.stdout: mon_down_mkfs_grace = 2m 2024-09-14T08:32:21.331 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:32:21.331 INFO:teuthology.orchestra.run.smithi164.stdout: mon_warn_on_filestore_osds = false 2024-09-14T08:32:21.331 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:32:21.331 INFO:teuthology.orchestra.run.smithi164.stdout:[client] 2024-09-14T08:32:21.331 INFO:teuthology.orchestra.run.smithi164.stdout: rgw cache enabled = true 2024-09-14T08:32:21.331 INFO:teuthology.orchestra.run.smithi164.stdout: rgw enable ops log = true 2024-09-14T08:32:21.331 INFO:teuthology.orchestra.run.smithi164.stdout: rgw enable usage log = true 2024-09-14T08:32:21.331 INFO:teuthology.orchestra.run.smithi164.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2024-09-14T08:32:21.331 INFO:teuthology.orchestra.run.smithi164.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2024-09-14T08:32:21.332 INFO:teuthology.orchestra.run.smithi164.stdout: debug rbd = 20 2024-09-14T08:32:21.332 INFO:teuthology.orchestra.run.smithi164.stdout: debug rbd_mirror = 15 2024-09-14T08:32:21.332 INFO:teuthology.orchestra.run.smithi164.stdout: log to stderr = False 2024-09-14T08:32:21.332 INFO:teuthology.orchestra.run.smithi164.stdout: rbd default features = 125 2024-09-14T08:32:21.332 INFO:teuthology.orchestra.run.smithi164.stdout: rbd mirror image policy type = simple 2024-09-14T08:32:21.332 INFO:teuthology.orchestra.run.smithi164.stdout:[mon.a] 2024-09-14T08:32:21.332 INFO:teuthology.orchestra.run.smithi164.stdout:[client.mirror.0] 2024-09-14T08:32:21.332 INFO:teuthology.orchestra.run.smithi164.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-14T08:32:21.332 INFO:teuthology.orchestra.run.smithi164.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-14T08:32:21.332 INFO:teuthology.orchestra.run.smithi164.stdout:[client.mirror.1] 2024-09-14T08:32:21.333 INFO:teuthology.orchestra.run.smithi164.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-14T08:32:21.333 INFO:teuthology.orchestra.run.smithi164.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-14T08:32:21.333 INFO:teuthology.orchestra.run.smithi164.stdout:[client.mirror.2] 2024-09-14T08:32:21.333 INFO:teuthology.orchestra.run.smithi164.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-14T08:32:21.333 INFO:teuthology.orchestra.run.smithi164.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-14T08:32:21.333 INFO:teuthology.orchestra.run.smithi164.stdout:[client.mirror.3] 2024-09-14T08:32:21.333 INFO:teuthology.orchestra.run.smithi164.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-14T08:32:21.333 INFO:teuthology.orchestra.run.smithi164.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-14T08:32:21.334 INFO:tasks.ceph:Creating admin key on cluster1.mon.a... 2024-09-14T08:32:21.335 DEBUG:teuthology.orchestra.run.smithi135:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *' /etc/ceph/cluster1.keyring 2024-09-14T08:32:21.429 INFO:tasks.ceph:Copying monmap to all nodes... 2024-09-14T08:32:21.429 DEBUG:teuthology.orchestra.run.smithi135:> set -ex 2024-09-14T08:32:21.429 DEBUG:teuthology.orchestra.run.smithi135:> dd if=/etc/ceph/cluster1.keyring of=/dev/stdout 2024-09-14T08:32:21.445 DEBUG:teuthology.orchestra.run.smithi135:> set -ex 2024-09-14T08:32:21.445 DEBUG:teuthology.orchestra.run.smithi135:> dd if=/home/ubuntu/cephtest/cluster1.monmap of=/dev/stdout 2024-09-14T08:32:21.501 INFO:tasks.ceph:Sending monmap to node ubuntu@smithi135.front.sepia.ceph.com 2024-09-14T08:32:21.501 DEBUG:teuthology.orchestra.run.smithi135:> set -ex 2024-09-14T08:32:21.501 DEBUG:teuthology.orchestra.run.smithi135:> sudo dd of=/etc/ceph/cluster1.keyring 2024-09-14T08:32:21.501 DEBUG:teuthology.orchestra.run.smithi135:> sudo chmod 0644 /etc/ceph/cluster1.keyring 2024-09-14T08:32:21.575 DEBUG:teuthology.orchestra.run.smithi135:> set -ex 2024-09-14T08:32:21.576 DEBUG:teuthology.orchestra.run.smithi135:> dd of=/home/ubuntu/cephtest/cluster1.monmap 2024-09-14T08:32:21.630 INFO:tasks.ceph:Sending monmap to node ubuntu@smithi164.front.sepia.ceph.com 2024-09-14T08:32:21.631 DEBUG:teuthology.orchestra.run.smithi164:> set -ex 2024-09-14T08:32:21.631 DEBUG:teuthology.orchestra.run.smithi164:> sudo dd of=/etc/ceph/cluster1.keyring 2024-09-14T08:32:21.631 DEBUG:teuthology.orchestra.run.smithi164:> sudo chmod 0644 /etc/ceph/cluster1.keyring 2024-09-14T08:32:21.671 DEBUG:teuthology.orchestra.run.smithi164:> set -ex 2024-09-14T08:32:21.672 DEBUG:teuthology.orchestra.run.smithi164:> dd of=/home/ubuntu/cephtest/cluster1.monmap 2024-09-14T08:32:21.729 INFO:tasks.ceph:Setting up mon nodes... 2024-09-14T08:32:21.730 INFO:tasks.ceph:Setting up mgr nodes... 2024-09-14T08:32:21.730 DEBUG:teuthology.orchestra.run.smithi135:> sudo mkdir -p /var/lib/ceph/mgr/cluster1-x && sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=mgr.x /var/lib/ceph/mgr/cluster1-x/keyring 2024-09-14T08:32:21.780 INFO:teuthology.orchestra.run.smithi135.stdout:creating /var/lib/ceph/mgr/cluster1-x/keyring 2024-09-14T08:32:21.783 INFO:tasks.ceph:Setting up mds nodes... 2024-09-14T08:32:21.783 INFO:tasks.ceph_client:Setting up client nodes... 2024-09-14T08:32:21.783 DEBUG:teuthology.orchestra.run.smithi135:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.0 /etc/ceph/cluster1.client.0.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.0.keyring 2024-09-14T08:32:21.825 INFO:teuthology.orchestra.run.smithi135.stdout:creating /etc/ceph/cluster1.client.0.keyring 2024-09-14T08:32:21.837 DEBUG:teuthology.orchestra.run.smithi164:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror /etc/ceph/cluster1.client.mirror.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.keyring 2024-09-14T08:32:21.896 INFO:teuthology.orchestra.run.smithi164.stdout:creating /etc/ceph/cluster1.client.mirror.keyring 2024-09-14T08:32:21.910 DEBUG:teuthology.orchestra.run.smithi164:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.0 /etc/ceph/cluster1.client.mirror.0.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.0.keyring 2024-09-14T08:32:21.953 INFO:teuthology.orchestra.run.smithi164.stdout:creating /etc/ceph/cluster1.client.mirror.0.keyring 2024-09-14T08:32:21.966 DEBUG:teuthology.orchestra.run.smithi164:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.1 /etc/ceph/cluster1.client.mirror.1.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.1.keyring 2024-09-14T08:32:22.049 INFO:teuthology.orchestra.run.smithi164.stdout:creating /etc/ceph/cluster1.client.mirror.1.keyring 2024-09-14T08:32:22.061 DEBUG:teuthology.orchestra.run.smithi164:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.2 /etc/ceph/cluster1.client.mirror.2.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.2.keyring 2024-09-14T08:32:22.142 INFO:teuthology.orchestra.run.smithi164.stdout:creating /etc/ceph/cluster1.client.mirror.2.keyring 2024-09-14T08:32:22.155 DEBUG:teuthology.orchestra.run.smithi164:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.3 /etc/ceph/cluster1.client.mirror.3.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.3.keyring 2024-09-14T08:32:22.237 INFO:teuthology.orchestra.run.smithi164.stdout:creating /etc/ceph/cluster1.client.mirror.3.keyring 2024-09-14T08:32:22.249 DEBUG:teuthology.orchestra.run.smithi164:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.4 /etc/ceph/cluster1.client.mirror.4.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.4.keyring 2024-09-14T08:32:22.331 INFO:teuthology.orchestra.run.smithi164.stdout:creating /etc/ceph/cluster1.client.mirror.4.keyring 2024-09-14T08:32:22.344 DEBUG:teuthology.orchestra.run.smithi164:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.5 /etc/ceph/cluster1.client.mirror.5.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.5.keyring 2024-09-14T08:32:22.425 INFO:teuthology.orchestra.run.smithi164.stdout:creating /etc/ceph/cluster1.client.mirror.5.keyring 2024-09-14T08:32:22.438 DEBUG:teuthology.orchestra.run.smithi164:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.6 /etc/ceph/cluster1.client.mirror.6.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.6.keyring 2024-09-14T08:32:22.519 INFO:teuthology.orchestra.run.smithi164.stdout:creating /etc/ceph/cluster1.client.mirror.6.keyring 2024-09-14T08:32:22.532 INFO:tasks.ceph:Running mkfs on osd nodes... 2024-09-14T08:32:22.532 INFO:tasks.ceph:ctx.disk_config.remote_to_roles_to_dev: {Remote(name='ubuntu@smithi135.front.sepia.ceph.com'): {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'}} 2024-09-14T08:32:22.532 DEBUG:teuthology.orchestra.run.smithi135:> sudo mkdir -p /var/lib/ceph/osd/cluster1-0 2024-09-14T08:32:22.559 INFO:tasks.ceph:roles_to_devs: {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'} 2024-09-14T08:32:22.559 INFO:tasks.ceph:role: cluster1.osd.0 2024-09-14T08:32:22.559 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_1 on ubuntu@smithi135.front.sepia.ceph.com 2024-09-14T08:32:22.559 DEBUG:teuthology.orchestra.run.smithi135:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_1 2024-09-14T08:32:22.633 INFO:teuthology.orchestra.run.smithi135.stdout:meta-data=/dev/vg_nvme/lv_1 isize=2048 agcount=4, agsize=5859072 blks 2024-09-14T08:32:22.633 INFO:teuthology.orchestra.run.smithi135.stdout: = sectsz=512 attr=2, projid32bit=1 2024-09-14T08:32:22.633 INFO:teuthology.orchestra.run.smithi135.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2024-09-14T08:32:22.633 INFO:teuthology.orchestra.run.smithi135.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2024-09-14T08:32:22.633 INFO:teuthology.orchestra.run.smithi135.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2024-09-14T08:32:22.633 INFO:teuthology.orchestra.run.smithi135.stdout: = sunit=0 swidth=0 blks 2024-09-14T08:32:22.633 INFO:teuthology.orchestra.run.smithi135.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2024-09-14T08:32:22.633 INFO:teuthology.orchestra.run.smithi135.stdout:log =internal log bsize=4096 blocks=16384, version=2 2024-09-14T08:32:22.633 INFO:teuthology.orchestra.run.smithi135.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2024-09-14T08:32:22.633 INFO:teuthology.orchestra.run.smithi135.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2024-09-14T08:32:23.125 INFO:teuthology.orchestra.run.smithi135.stdout:Discarding blocks...Done. 2024-09-14T08:32:23.127 INFO:tasks.ceph:mount /dev/vg_nvme/lv_1 on ubuntu@smithi135.front.sepia.ceph.com -o noatime 2024-09-14T08:32:23.127 DEBUG:teuthology.orchestra.run.smithi135:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_1 /var/lib/ceph/osd/cluster1-0 2024-09-14T08:32:23.269 DEBUG:teuthology.orchestra.run.smithi135:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster1-0 2024-09-14T08:32:23.302 DEBUG:teuthology.orchestra.run.smithi135:> sudo mkdir -p /var/lib/ceph/osd/cluster1-1 2024-09-14T08:32:23.369 INFO:tasks.ceph:roles_to_devs: {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'} 2024-09-14T08:32:23.369 INFO:tasks.ceph:role: cluster1.osd.1 2024-09-14T08:32:23.369 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_2 on ubuntu@smithi135.front.sepia.ceph.com 2024-09-14T08:32:23.369 DEBUG:teuthology.orchestra.run.smithi135:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_2 2024-09-14T08:32:23.444 INFO:teuthology.orchestra.run.smithi135.stdout:meta-data=/dev/vg_nvme/lv_2 isize=2048 agcount=4, agsize=5859072 blks 2024-09-14T08:32:23.445 INFO:teuthology.orchestra.run.smithi135.stdout: = sectsz=512 attr=2, projid32bit=1 2024-09-14T08:32:23.445 INFO:teuthology.orchestra.run.smithi135.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2024-09-14T08:32:23.445 INFO:teuthology.orchestra.run.smithi135.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2024-09-14T08:32:23.445 INFO:teuthology.orchestra.run.smithi135.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2024-09-14T08:32:23.445 INFO:teuthology.orchestra.run.smithi135.stdout: = sunit=0 swidth=0 blks 2024-09-14T08:32:23.445 INFO:teuthology.orchestra.run.smithi135.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2024-09-14T08:32:23.445 INFO:teuthology.orchestra.run.smithi135.stdout:log =internal log bsize=4096 blocks=16384, version=2 2024-09-14T08:32:23.445 INFO:teuthology.orchestra.run.smithi135.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2024-09-14T08:32:23.445 INFO:teuthology.orchestra.run.smithi135.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2024-09-14T08:32:23.898 INFO:teuthology.orchestra.run.smithi135.stdout:Discarding blocks...Done. 2024-09-14T08:32:23.900 INFO:tasks.ceph:mount /dev/vg_nvme/lv_2 on ubuntu@smithi135.front.sepia.ceph.com -o noatime 2024-09-14T08:32:23.900 DEBUG:teuthology.orchestra.run.smithi135:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_2 /var/lib/ceph/osd/cluster1-1 2024-09-14T08:32:23.994 DEBUG:teuthology.orchestra.run.smithi135:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster1-1 2024-09-14T08:32:24.022 DEBUG:teuthology.orchestra.run.smithi135:> sudo mkdir -p /var/lib/ceph/osd/cluster1-2 2024-09-14T08:32:24.088 INFO:tasks.ceph:roles_to_devs: {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'} 2024-09-14T08:32:24.088 INFO:tasks.ceph:role: cluster1.osd.2 2024-09-14T08:32:24.088 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_3 on ubuntu@smithi135.front.sepia.ceph.com 2024-09-14T08:32:24.088 DEBUG:teuthology.orchestra.run.smithi135:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_3 2024-09-14T08:32:24.160 INFO:teuthology.orchestra.run.smithi135.stdout:meta-data=/dev/vg_nvme/lv_3 isize=2048 agcount=4, agsize=5859072 blks 2024-09-14T08:32:24.160 INFO:teuthology.orchestra.run.smithi135.stdout: = sectsz=512 attr=2, projid32bit=1 2024-09-14T08:32:24.160 INFO:teuthology.orchestra.run.smithi135.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2024-09-14T08:32:24.161 INFO:teuthology.orchestra.run.smithi135.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2024-09-14T08:32:24.161 INFO:teuthology.orchestra.run.smithi135.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2024-09-14T08:32:24.161 INFO:teuthology.orchestra.run.smithi135.stdout: = sunit=0 swidth=0 blks 2024-09-14T08:32:24.161 INFO:teuthology.orchestra.run.smithi135.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2024-09-14T08:32:24.161 INFO:teuthology.orchestra.run.smithi135.stdout:log =internal log bsize=4096 blocks=16384, version=2 2024-09-14T08:32:24.161 INFO:teuthology.orchestra.run.smithi135.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2024-09-14T08:32:24.161 INFO:teuthology.orchestra.run.smithi135.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2024-09-14T08:32:24.661 INFO:teuthology.orchestra.run.smithi135.stdout:Discarding blocks...Done. 2024-09-14T08:32:24.663 INFO:tasks.ceph:mount /dev/vg_nvme/lv_3 on ubuntu@smithi135.front.sepia.ceph.com -o noatime 2024-09-14T08:32:24.664 DEBUG:teuthology.orchestra.run.smithi135:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_3 /var/lib/ceph/osd/cluster1-2 2024-09-14T08:32:24.786 DEBUG:teuthology.orchestra.run.smithi135:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster1-2 2024-09-14T08:32:24.815 DEBUG:teuthology.orchestra.run.smithi135:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster1 --mkfs --mkkey -i 0 --monmap /home/ubuntu/cephtest/cluster1.monmap 2024-09-14T08:32:24.911 INFO:teuthology.orchestra.run.smithi135.stderr:2024-09-14T08:32:24.911+0000 7f7187e0b640 -1 auth: error reading file: /var/lib/ceph/osd/cluster1-0/keyring: can't open /var/lib/ceph/osd/cluster1-0/keyring: (2) No such file or directory 2024-09-14T08:32:24.913 INFO:teuthology.orchestra.run.smithi135.stderr:2024-09-14T08:32:24.913+0000 7f7187e0b640 -1 created new key in keyring /var/lib/ceph/osd/cluster1-0/keyring 2024-09-14T08:32:24.913 INFO:teuthology.orchestra.run.smithi135.stderr:2024-09-14T08:32:24.913+0000 7f7187e0b640 -1 bdev(0x555a4991f800 /var/lib/ceph/osd/cluster1-0/block) open stat got: (1) Operation not permitted 2024-09-14T08:32:24.913 INFO:teuthology.orchestra.run.smithi135.stderr:2024-09-14T08:32:24.913+0000 7f7187e0b640 -1 bluestore(/var/lib/ceph/osd/cluster1-0) _read_fsid unparsable uuid 2024-09-14T08:32:28.064 DEBUG:teuthology.orchestra.run.smithi135:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster1-0 2024-09-14T08:32:28.091 DEBUG:teuthology.orchestra.run.smithi135:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster1 --mkfs --mkkey -i 1 --monmap /home/ubuntu/cephtest/cluster1.monmap 2024-09-14T08:32:28.179 INFO:teuthology.orchestra.run.smithi135.stderr:2024-09-14T08:32:28.179+0000 7f3f7d018640 -1 auth: error reading file: /var/lib/ceph/osd/cluster1-1/keyring: can't open /var/lib/ceph/osd/cluster1-1/keyring: (2) No such file or directory 2024-09-14T08:32:28.179 INFO:teuthology.orchestra.run.smithi135.stderr:2024-09-14T08:32:28.180+0000 7f3f7d018640 -1 created new key in keyring /var/lib/ceph/osd/cluster1-1/keyring 2024-09-14T08:32:28.180 INFO:teuthology.orchestra.run.smithi135.stderr:2024-09-14T08:32:28.180+0000 7f3f7d018640 -1 bdev(0x55be0d989800 /var/lib/ceph/osd/cluster1-1/block) open stat got: (1) Operation not permitted 2024-09-14T08:32:28.180 INFO:teuthology.orchestra.run.smithi135.stderr:2024-09-14T08:32:28.180+0000 7f3f7d018640 -1 bluestore(/var/lib/ceph/osd/cluster1-1) _read_fsid unparsable uuid 2024-09-14T08:32:31.329 DEBUG:teuthology.orchestra.run.smithi135:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster1-1 2024-09-14T08:32:31.357 DEBUG:teuthology.orchestra.run.smithi135:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster1 --mkfs --mkkey -i 2 --monmap /home/ubuntu/cephtest/cluster1.monmap 2024-09-14T08:32:31.446 INFO:teuthology.orchestra.run.smithi135.stderr:2024-09-14T08:32:31.446+0000 7f248e60d640 -1 auth: error reading file: /var/lib/ceph/osd/cluster1-2/keyring: can't open /var/lib/ceph/osd/cluster1-2/keyring: (2) No such file or directory 2024-09-14T08:32:31.446 INFO:teuthology.orchestra.run.smithi135.stderr:2024-09-14T08:32:31.446+0000 7f248e60d640 -1 created new key in keyring /var/lib/ceph/osd/cluster1-2/keyring 2024-09-14T08:32:31.446 INFO:teuthology.orchestra.run.smithi135.stderr:2024-09-14T08:32:31.446+0000 7f248e60d640 -1 bdev(0x5572e1e87800 /var/lib/ceph/osd/cluster1-2/block) open stat got: (1) Operation not permitted 2024-09-14T08:32:31.446 INFO:teuthology.orchestra.run.smithi135.stderr:2024-09-14T08:32:31.446+0000 7f248e60d640 -1 bluestore(/var/lib/ceph/osd/cluster1-2) _read_fsid unparsable uuid 2024-09-14T08:32:34.356 DEBUG:teuthology.orchestra.run.smithi135:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster1-2 2024-09-14T08:32:34.383 INFO:tasks.ceph:Reading keys from all nodes... 2024-09-14T08:32:34.383 DEBUG:teuthology.orchestra.run.smithi135:> set -ex 2024-09-14T08:32:34.383 DEBUG:teuthology.orchestra.run.smithi135:> sudo dd if=/var/lib/ceph/mgr/cluster1-x/keyring of=/dev/stdout 2024-09-14T08:32:34.450 DEBUG:teuthology.orchestra.run.smithi135:> set -ex 2024-09-14T08:32:34.450 DEBUG:teuthology.orchestra.run.smithi135:> sudo dd if=/var/lib/ceph/osd/cluster1-0/keyring of=/dev/stdout 2024-09-14T08:32:34.516 DEBUG:teuthology.orchestra.run.smithi135:> set -ex 2024-09-14T08:32:34.516 DEBUG:teuthology.orchestra.run.smithi135:> sudo dd if=/var/lib/ceph/osd/cluster1-1/keyring of=/dev/stdout 2024-09-14T08:32:34.582 DEBUG:teuthology.orchestra.run.smithi135:> set -ex 2024-09-14T08:32:34.582 DEBUG:teuthology.orchestra.run.smithi135:> sudo dd if=/var/lib/ceph/osd/cluster1-2/keyring of=/dev/stdout 2024-09-14T08:32:34.648 DEBUG:teuthology.orchestra.run.smithi135:> set -ex 2024-09-14T08:32:34.648 DEBUG:teuthology.orchestra.run.smithi135:> dd if=/etc/ceph/cluster1.client.0.keyring of=/dev/stdout 2024-09-14T08:32:34.703 DEBUG:teuthology.orchestra.run.smithi164:> set -ex 2024-09-14T08:32:34.703 DEBUG:teuthology.orchestra.run.smithi164:> dd if=/etc/ceph/cluster1.client.mirror.keyring of=/dev/stdout 2024-09-14T08:32:34.723 DEBUG:teuthology.orchestra.run.smithi164:> set -ex 2024-09-14T08:32:34.723 DEBUG:teuthology.orchestra.run.smithi164:> dd if=/etc/ceph/cluster1.client.mirror.0.keyring of=/dev/stdout 2024-09-14T08:32:34.782 DEBUG:teuthology.orchestra.run.smithi164:> set -ex 2024-09-14T08:32:34.782 DEBUG:teuthology.orchestra.run.smithi164:> dd if=/etc/ceph/cluster1.client.mirror.1.keyring of=/dev/stdout 2024-09-14T08:32:34.841 DEBUG:teuthology.orchestra.run.smithi164:> set -ex 2024-09-14T08:32:34.841 DEBUG:teuthology.orchestra.run.smithi164:> dd if=/etc/ceph/cluster1.client.mirror.2.keyring of=/dev/stdout 2024-09-14T08:32:34.899 DEBUG:teuthology.orchestra.run.smithi164:> set -ex 2024-09-14T08:32:34.900 DEBUG:teuthology.orchestra.run.smithi164:> dd if=/etc/ceph/cluster1.client.mirror.3.keyring of=/dev/stdout 2024-09-14T08:32:34.958 DEBUG:teuthology.orchestra.run.smithi164:> set -ex 2024-09-14T08:32:34.958 DEBUG:teuthology.orchestra.run.smithi164:> dd if=/etc/ceph/cluster1.client.mirror.4.keyring of=/dev/stdout 2024-09-14T08:32:35.014 DEBUG:teuthology.orchestra.run.smithi164:> set -ex 2024-09-14T08:32:35.014 DEBUG:teuthology.orchestra.run.smithi164:> dd if=/etc/ceph/cluster1.client.mirror.5.keyring of=/dev/stdout 2024-09-14T08:32:35.071 DEBUG:teuthology.orchestra.run.smithi164:> set -ex 2024-09-14T08:32:35.071 DEBUG:teuthology.orchestra.run.smithi164:> dd if=/etc/ceph/cluster1.client.mirror.6.keyring of=/dev/stdout 2024-09-14T08:32:35.128 INFO:tasks.ceph:Adding keys to all mons... 2024-09-14T08:32:35.128 DEBUG:teuthology.orchestra.run.smithi135:> sudo tee -a /etc/ceph/cluster1.keyring 2024-09-14T08:32:35.151 INFO:teuthology.orchestra.run.smithi135.stdout:[mgr.x] 2024-09-14T08:32:35.151 INFO:teuthology.orchestra.run.smithi135.stdout: key = AQAVSuVm1CqgLhAAk/8G2Phmc0PbBFcYGsNOqw== 2024-09-14T08:32:35.151 INFO:teuthology.orchestra.run.smithi135.stdout:[osd.0] 2024-09-14T08:32:35.152 INFO:teuthology.orchestra.run.smithi135.stdout: key = AQAYSuVmRjtnNhAAg+cw2QtkhsBqVwttAZlXEA== 2024-09-14T08:32:35.152 INFO:teuthology.orchestra.run.smithi135.stdout:[osd.1] 2024-09-14T08:32:35.152 INFO:teuthology.orchestra.run.smithi135.stdout: key = AQAcSuVmn3rEChAADv+8oK1PE1BKwwPNjnLWFA== 2024-09-14T08:32:35.152 INFO:teuthology.orchestra.run.smithi135.stdout:[osd.2] 2024-09-14T08:32:35.152 INFO:teuthology.orchestra.run.smithi135.stdout: key = AQAfSuVmYUGoGhAAXYnkPa3ndY0LhFfknAL+zg== 2024-09-14T08:32:35.152 INFO:teuthology.orchestra.run.smithi135.stdout:[client.0] 2024-09-14T08:32:35.152 INFO:teuthology.orchestra.run.smithi135.stdout: key = AQAVSuVm4s5AMRAAxiyP0ejn/4OWCGGAlsWutQ== 2024-09-14T08:32:35.152 INFO:teuthology.orchestra.run.smithi135.stdout:[client.mirror] 2024-09-14T08:32:35.152 INFO:teuthology.orchestra.run.smithi135.stdout: key = AQAVSuVmNEp6NRAAd0nGGnnjv95u4Lypkcb1Sg== 2024-09-14T08:32:35.152 INFO:teuthology.orchestra.run.smithi135.stdout:[client.mirror.0] 2024-09-14T08:32:35.152 INFO:teuthology.orchestra.run.smithi135.stdout: key = AQAVSuVmypPsOBAA5ZqBI4/dRwbC2kbumMna0Q== 2024-09-14T08:32:35.152 INFO:teuthology.orchestra.run.smithi135.stdout:[client.mirror.1] 2024-09-14T08:32:35.153 INFO:teuthology.orchestra.run.smithi135.stdout: key = AQAWSuVmpQn7AhAAw10ZlA8g9gGLtqer42Mv2g== 2024-09-14T08:32:35.153 INFO:teuthology.orchestra.run.smithi135.stdout:[client.mirror.2] 2024-09-14T08:32:35.153 INFO:teuthology.orchestra.run.smithi135.stdout: key = AQAWSuVmVNGVCBAAJeLJHUqgK83KJhmZFWVyjw== 2024-09-14T08:32:35.153 INFO:teuthology.orchestra.run.smithi135.stdout:[client.mirror.3] 2024-09-14T08:32:35.153 INFO:teuthology.orchestra.run.smithi135.stdout: key = AQAWSuVmQqEvDhAAHf4QjU2RjJVS5hfgppC5fg== 2024-09-14T08:32:35.153 INFO:teuthology.orchestra.run.smithi135.stdout:[client.mirror.4] 2024-09-14T08:32:35.153 INFO:teuthology.orchestra.run.smithi135.stdout: key = AQAWSuVmrejaExAA3TxY94t8vTexj81A3lV/ow== 2024-09-14T08:32:35.153 INFO:teuthology.orchestra.run.smithi135.stdout:[client.mirror.5] 2024-09-14T08:32:35.153 INFO:teuthology.orchestra.run.smithi135.stdout: key = AQAWSuVmsrtxGRAArD2nlrElLw7woZ0DjKT7Sw== 2024-09-14T08:32:35.153 INFO:teuthology.orchestra.run.smithi135.stdout:[client.mirror.6] 2024-09-14T08:32:35.153 INFO:teuthology.orchestra.run.smithi135.stdout: key = AQAWSuVm3vQLHxAAKu1JARgM9cotPa9b2purFg== 2024-09-14T08:32:35.154 DEBUG:teuthology.orchestra.run.smithi135:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=mgr.x --cap mon 'allow profile mgr' --cap osd 'allow *' --cap mds 'allow *' 2024-09-14T08:32:35.246 DEBUG:teuthology.orchestra.run.smithi135:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=osd.0 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2024-09-14T08:32:35.297 DEBUG:teuthology.orchestra.run.smithi135:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=osd.1 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2024-09-14T08:32:35.348 DEBUG:teuthology.orchestra.run.smithi135:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=osd.2 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2024-09-14T08:32:35.399 DEBUG:teuthology.orchestra.run.smithi135:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-09-14T08:32:35.453 DEBUG:teuthology.orchestra.run.smithi135:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-09-14T08:32:35.507 DEBUG:teuthology.orchestra.run.smithi135:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-09-14T08:32:35.560 DEBUG:teuthology.orchestra.run.smithi135:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.1 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-09-14T08:32:35.615 DEBUG:teuthology.orchestra.run.smithi135:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.2 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-09-14T08:32:35.670 DEBUG:teuthology.orchestra.run.smithi135:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.3 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-09-14T08:32:35.724 DEBUG:teuthology.orchestra.run.smithi135:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.4 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-09-14T08:32:35.778 DEBUG:teuthology.orchestra.run.smithi135:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.5 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-09-14T08:32:35.833 DEBUG:teuthology.orchestra.run.smithi135:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.6 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-09-14T08:32:35.887 INFO:tasks.ceph:Running mkfs on mon nodes... 2024-09-14T08:32:35.887 DEBUG:teuthology.orchestra.run.smithi135:> sudo mkdir -p /var/lib/ceph/mon/cluster1-a 2024-09-14T08:32:35.912 DEBUG:teuthology.orchestra.run.smithi135:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-mon --cluster cluster1 --mkfs -i a --monmap /home/ubuntu/cephtest/cluster1.monmap --keyring /etc/ceph/cluster1.keyring 2024-09-14T08:32:36.008 DEBUG:teuthology.orchestra.run.smithi135:> sudo chown -R ceph:ceph /var/lib/ceph/mon/cluster1-a 2024-09-14T08:32:36.032 DEBUG:teuthology.orchestra.run.smithi135:> rm -- /home/ubuntu/cephtest/cluster1.monmap 2024-09-14T08:32:36.087 INFO:tasks.ceph:Starting mon daemons in cluster cluster1... 2024-09-14T08:32:36.088 INFO:tasks.ceph.cluster1.mon.a:Restarting daemon 2024-09-14T08:32:36.088 DEBUG:teuthology.orchestra.run.smithi135:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f --cluster cluster1 -i a 2024-09-14T08:32:36.129 INFO:tasks.ceph.cluster1.mon.a:Started 2024-09-14T08:32:36.130 INFO:tasks.ceph:Starting mgr daemons in cluster cluster1... 2024-09-14T08:32:36.130 INFO:tasks.ceph.cluster1.mgr.x:Restarting daemon 2024-09-14T08:32:36.130 DEBUG:teuthology.orchestra.run.smithi135:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mgr -f --cluster cluster1 -i x 2024-09-14T08:32:36.132 INFO:tasks.ceph.cluster1.mgr.x:Started 2024-09-14T08:32:36.132 DEBUG:tasks.ceph:set 0 configs 2024-09-14T08:32:36.132 INFO:tasks.ceph:Setting crush tunables to default 2024-09-14T08:32:36.133 DEBUG:teuthology.orchestra.run.smithi135:> sudo ceph --cluster cluster1 osd crush tunables default 2024-09-14T08:32:36.305 INFO:teuthology.orchestra.run.smithi135.stderr:adjusted tunables profile to default 2024-09-14T08:32:36.316 INFO:tasks.ceph:check_enable_crimson: False 2024-09-14T08:32:36.317 INFO:tasks.ceph:Starting osd daemons in cluster cluster1... 2024-09-14T08:32:36.317 DEBUG:teuthology.orchestra.run.smithi135:> set -ex 2024-09-14T08:32:36.317 DEBUG:teuthology.orchestra.run.smithi135:> sudo dd if=/var/lib/ceph/osd/cluster1-0/fsid of=/dev/stdout 2024-09-14T08:32:36.341 DEBUG:teuthology.orchestra.run.smithi135:> set -ex 2024-09-14T08:32:36.342 DEBUG:teuthology.orchestra.run.smithi135:> sudo dd if=/var/lib/ceph/osd/cluster1-1/fsid of=/dev/stdout 2024-09-14T08:32:36.406 DEBUG:teuthology.orchestra.run.smithi135:> set -ex 2024-09-14T08:32:36.406 DEBUG:teuthology.orchestra.run.smithi135:> sudo dd if=/var/lib/ceph/osd/cluster1-2/fsid of=/dev/stdout 2024-09-14T08:32:36.472 DEBUG:teuthology.orchestra.run.smithi135:> sudo ceph --cluster cluster1 osd new 72211b81-bcd2-4a02-a026-a5ee8497a3de 0 2024-09-14T08:32:36.693 INFO:teuthology.orchestra.run.smithi135.stdout:0 2024-09-14T08:32:36.705 DEBUG:teuthology.orchestra.run.smithi135:> sudo ceph --cluster cluster1 osd new fed7a36b-99f5-45e7-8984-de5e09fba654 1 2024-09-14T08:32:36.882 INFO:teuthology.orchestra.run.smithi135.stdout:1 2024-09-14T08:32:36.894 DEBUG:teuthology.orchestra.run.smithi135:> sudo ceph --cluster cluster1 osd new f39a5a76-8dc4-46f2-9e8d-c490677056f6 2 2024-09-14T08:32:37.071 INFO:teuthology.orchestra.run.smithi135.stdout:2 2024-09-14T08:32:37.082 INFO:tasks.ceph.cluster1.osd.0:Restarting daemon 2024-09-14T08:32:37.083 DEBUG:teuthology.orchestra.run.smithi135:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster1 -i 0 2024-09-14T08:32:37.085 INFO:tasks.ceph.cluster1.osd.0:Started 2024-09-14T08:32:37.085 INFO:tasks.ceph.cluster1.osd.1:Restarting daemon 2024-09-14T08:32:37.085 DEBUG:teuthology.orchestra.run.smithi135:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster1 -i 1 2024-09-14T08:32:37.087 INFO:tasks.ceph.cluster1.osd.1:Started 2024-09-14T08:32:37.087 INFO:tasks.ceph.cluster1.osd.2:Restarting daemon 2024-09-14T08:32:37.087 DEBUG:teuthology.orchestra.run.smithi135:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster1 -i 2 2024-09-14T08:32:37.090 INFO:tasks.ceph.cluster1.osd.2:Started 2024-09-14T08:32:37.090 DEBUG:teuthology.orchestra.run.smithi135:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd dump --format=json 2024-09-14T08:32:37.280 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:32:37.280 INFO:teuthology.orchestra.run.smithi135.stdout:{"epoch":5,"fsid":"77c3c5ea-bdba-488a-b29d-8702bc143d62","created":"2024-09-14T08:32:36.219895+0000","modified":"2024-09-14T08:32:37.071724+0000","last_up_change":"0.000000","last_in_change":"2024-09-14T08:32:37.071724+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"72211b81-bcd2-4a02-a026-a5ee8497a3de","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"fed7a36b-99f5-45e7-8984-de5e09fba654","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"f39a5a76-8dc4-46f2-9e8d-c490677056f6","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-14T08:32:37.291 INFO:tasks.ceph.ceph_manager.cluster1:[] 2024-09-14T08:32:37.291 INFO:tasks.ceph:Waiting for OSDs to come up 2024-09-14T08:32:37.594 DEBUG:teuthology.orchestra.run.smithi135:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster1 osd dump --format=json 2024-09-14T08:32:37.887 INFO:teuthology.misc.health.smithi135.stderr:2024-09-14T08:32:37.888+0000 7f6985a21640 0 --2- 172.21.15.135:0/3978634858 >> v2:172.21.15.135:3300/0 conn(0x7f6980073380 0x7f6980071f30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request get_initial_auth_request returned -2 2024-09-14T08:32:37.955 INFO:teuthology.misc.health.smithi135.stdout: 2024-09-14T08:32:37.955 INFO:teuthology.misc.health.smithi135.stdout:{"epoch":5,"fsid":"77c3c5ea-bdba-488a-b29d-8702bc143d62","created":"2024-09-14T08:32:36.219895+0000","modified":"2024-09-14T08:32:37.071724+0000","last_up_change":"0.000000","last_in_change":"2024-09-14T08:32:37.071724+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"72211b81-bcd2-4a02-a026-a5ee8497a3de","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"fed7a36b-99f5-45e7-8984-de5e09fba654","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"f39a5a76-8dc4-46f2-9e8d-c490677056f6","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-14T08:32:37.965 DEBUG:teuthology.misc:0 of 3 OSDs are up 2024-09-14T08:32:38.538 INFO:tasks.ceph.cluster1.osd.0.smithi135.stderr:2024-09-14T08:32:38.539+0000 7f31f7064640 -1 Falling back to public interface 2024-09-14T08:32:38.539 INFO:tasks.ceph.cluster1.osd.2.smithi135.stderr:2024-09-14T08:32:38.539+0000 7f31888ef640 -1 Falling back to public interface 2024-09-14T08:32:38.821 INFO:tasks.ceph.cluster1.osd.1.smithi135.stderr:2024-09-14T08:32:38.821+0000 7f3b3b824640 -1 Falling back to public interface 2024-09-14T08:32:41.862 INFO:tasks.ceph.cluster1.mgr.x.smithi135.stderr:2024-09-14T08:32:41.863+0000 7fd11cf46640 -1 mgr.server handle_report got status from non-daemon mon.a 2024-09-14T08:32:42.326 INFO:tasks.ceph.cluster1.osd.2.smithi135.stderr:2024-09-14T08:32:42.326+0000 7f31888ef640 -1 osd.2 0 log_to_monitors true 2024-09-14T08:32:42.417 INFO:tasks.ceph.cluster1.osd.0.smithi135.stderr:2024-09-14T08:32:42.417+0000 7f31f7064640 -1 osd.0 0 log_to_monitors true 2024-09-14T08:32:42.888 INFO:tasks.ceph.cluster1.osd.1.smithi135.stderr:2024-09-14T08:32:42.888+0000 7f3b3b824640 -1 osd.1 0 log_to_monitors true 2024-09-14T08:32:44.268 DEBUG:teuthology.orchestra.run.smithi135:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster1 osd dump --format=json 2024-09-14T08:32:44.562 INFO:teuthology.misc.health.smithi135.stdout: 2024-09-14T08:32:44.562 INFO:teuthology.misc.health.smithi135.stdout:{"epoch":7,"fsid":"77c3c5ea-bdba-488a-b29d-8702bc143d62","created":"2024-09-14T08:32:36.219895+0000","modified":"2024-09-14T08:32:43.655717+0000","last_up_change":"0.000000","last_in_change":"2024-09-14T08:32:37.071724+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"72211b81-bcd2-4a02-a026-a5ee8497a3de","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"fed7a36b-99f5-45e7-8984-de5e09fba654","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"f39a5a76-8dc4-46f2-9e8d-c490677056f6","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-14T08:32:44.573 DEBUG:teuthology.misc:0 of 3 OSDs are up 2024-09-14T08:32:46.625 INFO:tasks.ceph.cluster1.osd.0.smithi135.stderr:2024-09-14T08:32:46.625+0000 7f31f2ff3640 -1 osd.0 0 waiting for initial osdmap 2024-09-14T08:32:46.637 INFO:tasks.ceph.cluster1.osd.0.smithi135.stderr:2024-09-14T08:32:46.638+0000 7f31eae12640 -1 osd.0 8 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-14T08:32:46.677 INFO:tasks.ceph.cluster1.osd.2.smithi135.stderr:2024-09-14T08:32:46.677+0000 7f318487e640 -1 osd.2 0 waiting for initial osdmap 2024-09-14T08:32:46.695 INFO:tasks.ceph.cluster1.osd.2.smithi135.stderr:2024-09-14T08:32:46.696+0000 7f317c69d640 -1 osd.2 8 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-14T08:32:47.568 INFO:tasks.ceph.cluster1.osd.1.smithi135.stderr:2024-09-14T08:32:47.568+0000 7f3b37a23640 -1 osd.1 0 waiting for initial osdmap 2024-09-14T08:32:47.577 INFO:tasks.ceph.cluster1.osd.1.smithi135.stderr:2024-09-14T08:32:47.578+0000 7f3b2f02e640 -1 osd.1 8 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-14T08:32:50.876 DEBUG:teuthology.orchestra.run.smithi135:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster1 osd dump --format=json 2024-09-14T08:32:51.174 INFO:teuthology.misc.health.smithi135.stdout: 2024-09-14T08:32:51.174 INFO:teuthology.misc.health.smithi135.stdout:{"epoch":12,"fsid":"77c3c5ea-bdba-488a-b29d-8702bc143d62","created":"2024-09-14T08:32:36.219895+0000","modified":"2024-09-14T08:32:50.679366+0000","last_up_change":"2024-09-14T08:32:47.627053+0000","last_in_change":"2024-09-14T08:32:37.071724+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-14T08:32:48.810068+0000","flags":32769,"flags_names":"hashpspool,creating","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"72211b81-bcd2-4a02-a026-a5ee8497a3de","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6800","nonce":1099719938},{"type":"v1","addr":"172.21.15.135:6801","nonce":1099719938}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6802","nonce":1099719938},{"type":"v1","addr":"172.21.15.135:6803","nonce":1099719938}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6806","nonce":1099719938},{"type":"v1","addr":"172.21.15.135:6808","nonce":1099719938}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6804","nonce":1099719938},{"type":"v1","addr":"172.21.15.135:6805","nonce":1099719938}]},"public_addr":"172.21.15.135:6801/1099719938","cluster_addr":"172.21.15.135:6803/1099719938","heartbeat_back_addr":"172.21.15.135:6808/1099719938","heartbeat_front_addr":"172.21.15.135:6805/1099719938","state":["exists","up"]},{"osd":1,"uuid":"fed7a36b-99f5-45e7-8984-de5e09fba654","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":11,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6816","nonce":1622923706},{"type":"v1","addr":"172.21.15.135:6817","nonce":1622923706}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6818","nonce":1622923706},{"type":"v1","addr":"172.21.15.135:6819","nonce":1622923706}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6822","nonce":1622923706},{"type":"v1","addr":"172.21.15.135:6823","nonce":1622923706}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6820","nonce":1622923706},{"type":"v1","addr":"172.21.15.135:6821","nonce":1622923706}]},"public_addr":"172.21.15.135:6817/1622923706","cluster_addr":"172.21.15.135:6819/1622923706","heartbeat_back_addr":"172.21.15.135:6823/1622923706","heartbeat_front_addr":"172.21.15.135:6821/1622923706","state":["exists","up"]},{"osd":2,"uuid":"f39a5a76-8dc4-46f2-9e8d-c490677056f6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6807","nonce":1946554517},{"type":"v1","addr":"172.21.15.135:6809","nonce":1946554517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6810","nonce":1946554517},{"type":"v1","addr":"172.21.15.135:6811","nonce":1946554517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6814","nonce":1946554517},{"type":"v1","addr":"172.21.15.135:6815","nonce":1946554517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6812","nonce":1946554517},{"type":"v1","addr":"172.21.15.135:6813","nonce":1946554517}]},"public_addr":"172.21.15.135:6809/1946554517","cluster_addr":"172.21.15.135:6811/1946554517","heartbeat_back_addr":"172.21.15.135:6815/1946554517","heartbeat_front_addr":"172.21.15.135:6813/1946554517","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T08:32:43.433705+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T08:32:43.895674+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T08:32:43.286080+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-14T08:32:51.186 DEBUG:teuthology.misc:3 of 3 OSDs are up 2024-09-14T08:32:51.186 INFO:tasks.ceph:Creating RBD pool 2024-09-14T08:32:51.186 DEBUG:teuthology.orchestra.run.smithi135:> sudo ceph --cluster cluster1 osd pool create rbd 8 2024-09-14T08:32:51.683 INFO:teuthology.orchestra.run.smithi135.stderr:pool 'rbd' created 2024-09-14T08:32:51.697 DEBUG:teuthology.orchestra.run.smithi135:> rbd --cluster cluster1 pool init rbd 2024-09-14T08:32:54.699 INFO:tasks.ceph:Starting mds daemons in cluster cluster1... 2024-09-14T08:32:54.700 DEBUG:teuthology.orchestra.run.smithi135:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 config log 1 --format=json 2024-09-14T08:32:54.700 INFO:tasks.daemonwatchdog.daemon_watchdog:watchdog starting 2024-09-14T08:32:54.991 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:32:55.004 INFO:teuthology.orchestra.run.smithi135.stdout:[{"version":4,"timestamp":"2024-09-14T08:32:47.569720+0000","name":"","changes":[{"name":"osd.1/osd_mclock_max_capacity_iops_ssd","new_value":"7465.422403"}]}] 2024-09-14T08:32:55.005 INFO:tasks.ceph_manager:config epoch is 4 2024-09-14T08:32:55.005 INFO:tasks.ceph:Waiting until cluster1 daemons up and pgs clean... 2024-09-14T08:32:55.005 INFO:tasks.ceph.ceph_manager.cluster1:waiting for mgr available 2024-09-14T08:32:55.005 DEBUG:teuthology.orchestra.run.smithi135:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 mgr dump --format=json 2024-09-14T08:32:55.335 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:32:55.346 INFO:teuthology.orchestra.run.smithi135.stdout:{"epoch":5,"flags":0,"active_gid":4102,"active_name":"x","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6824","nonce":1689850382},{"type":"v1","addr":"172.21.15.135:6825","nonce":1689850382}]},"active_addr":"172.21.15.135:6825/1689850382","active_change":"2024-09-14T08:32:40.588663+0000","active_mgr_features":4540719139924082687,"available":true,"standbys":[],"modules":["iostat","nfs","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to, use commas to separate multiple","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.27.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:10.4.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_nginx":{"name":"container_image_nginx","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nginx:1.26.1","min":"","max":"","enum_allowed":[],"desc":"Nginx container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.7.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.2.17","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_oauth2_proxy":{"name":"container_image_oauth2_proxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/oauth2-proxy/oauth2-proxy:v7.6.0","min":"","max":"","enum_allowed":[],"desc":"oauth2-proxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.51.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba":{"name":"container_image_samba","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-server:devbuilds-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba/SMB container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MANAGED_BY_CLUSTERS":{"name":"MANAGED_BY_CLUSTERS","type":"str","level":"advanced","flags":0,"default_value":"[]","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MULTICLUSTER_CONFIG":{"name":"MULTICLUSTER_CONFIG","type":"str","level":"advanced","flags":0,"default_value":"{}","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_requests":{"name":"max_requests","type":"int","level":"advanced","flags":0,"default_value":"500","min":"","max":"","enum_allowed":[],"desc":"Maximum number of requests to keep in memory. When new request comes in, the oldest request will be removed if the number of requests exceeds the max request number.if un-finished request is removed, error message will be logged in the ceph-mgr log.","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"secondary_zone_period_retry_limit":{"name":"secondary_zone_period_retry_limit","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"RGW module period update retry limit for secondary site","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"smb","can_run":true,"error_string":"","module_options":{"internal_store_backend":{"name":"internal_store_backend","type":"str","level":"dev","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"set internal store backend. for develoment and testing only","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"update_orchestration":{"name":"update_orchestration","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically update orchestration when smb resources are changed","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"squid":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":0,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.135:0","nonce":988681967}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.135:0","nonce":474624554}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.135:0","nonce":3687775772}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.135:0","nonce":1920319072}]}]} 2024-09-14T08:32:55.348 INFO:tasks.ceph.ceph_manager.cluster1:mgr available! 2024-09-14T08:32:55.348 INFO:tasks.ceph.ceph_manager.cluster1:waiting for all up 2024-09-14T08:32:55.349 DEBUG:teuthology.orchestra.run.smithi135:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd dump --format=json 2024-09-14T08:32:55.633 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:32:55.633 INFO:teuthology.orchestra.run.smithi135.stdout:{"epoch":16,"fsid":"77c3c5ea-bdba-488a-b29d-8702bc143d62","created":"2024-09-14T08:32:36.219895+0000","modified":"2024-09-14T08:32:54.686267+0000","last_up_change":"2024-09-14T08:32:47.627053+0000","last_in_change":"2024-09-14T08:32:37.071724+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-14T08:32:48.810068+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-14T08:32:51.489753+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":16,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"72211b81-bcd2-4a02-a026-a5ee8497a3de","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6800","nonce":1099719938},{"type":"v1","addr":"172.21.15.135:6801","nonce":1099719938}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6802","nonce":1099719938},{"type":"v1","addr":"172.21.15.135:6803","nonce":1099719938}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6806","nonce":1099719938},{"type":"v1","addr":"172.21.15.135:6808","nonce":1099719938}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6804","nonce":1099719938},{"type":"v1","addr":"172.21.15.135:6805","nonce":1099719938}]},"public_addr":"172.21.15.135:6801/1099719938","cluster_addr":"172.21.15.135:6803/1099719938","heartbeat_back_addr":"172.21.15.135:6808/1099719938","heartbeat_front_addr":"172.21.15.135:6805/1099719938","state":["exists","up"]},{"osd":1,"uuid":"fed7a36b-99f5-45e7-8984-de5e09fba654","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6816","nonce":1622923706},{"type":"v1","addr":"172.21.15.135:6817","nonce":1622923706}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6818","nonce":1622923706},{"type":"v1","addr":"172.21.15.135:6819","nonce":1622923706}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6822","nonce":1622923706},{"type":"v1","addr":"172.21.15.135:6823","nonce":1622923706}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6820","nonce":1622923706},{"type":"v1","addr":"172.21.15.135:6821","nonce":1622923706}]},"public_addr":"172.21.15.135:6817/1622923706","cluster_addr":"172.21.15.135:6819/1622923706","heartbeat_back_addr":"172.21.15.135:6823/1622923706","heartbeat_front_addr":"172.21.15.135:6821/1622923706","state":["exists","up"]},{"osd":2,"uuid":"f39a5a76-8dc4-46f2-9e8d-c490677056f6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6807","nonce":1946554517},{"type":"v1","addr":"172.21.15.135:6809","nonce":1946554517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6810","nonce":1946554517},{"type":"v1","addr":"172.21.15.135:6811","nonce":1946554517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6814","nonce":1946554517},{"type":"v1","addr":"172.21.15.135:6815","nonce":1946554517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6812","nonce":1946554517},{"type":"v1","addr":"172.21.15.135:6813","nonce":1946554517}]},"public_addr":"172.21.15.135:6809/1946554517","cluster_addr":"172.21.15.135:6811/1946554517","heartbeat_back_addr":"172.21.15.135:6815/1946554517","heartbeat_front_addr":"172.21.15.135:6813/1946554517","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T08:32:43.433705+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T08:32:43.895674+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T08:32:43.286080+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-14T08:32:55.646 INFO:tasks.ceph.ceph_manager.cluster1:all up! 2024-09-14T08:32:55.646 DEBUG:teuthology.orchestra.run.smithi135:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd dump --format=json 2024-09-14T08:32:55.928 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:32:55.928 INFO:teuthology.orchestra.run.smithi135.stdout:{"epoch":16,"fsid":"77c3c5ea-bdba-488a-b29d-8702bc143d62","created":"2024-09-14T08:32:36.219895+0000","modified":"2024-09-14T08:32:54.686267+0000","last_up_change":"2024-09-14T08:32:47.627053+0000","last_in_change":"2024-09-14T08:32:37.071724+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-14T08:32:48.810068+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-14T08:32:51.489753+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":16,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"72211b81-bcd2-4a02-a026-a5ee8497a3de","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6800","nonce":1099719938},{"type":"v1","addr":"172.21.15.135:6801","nonce":1099719938}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6802","nonce":1099719938},{"type":"v1","addr":"172.21.15.135:6803","nonce":1099719938}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6806","nonce":1099719938},{"type":"v1","addr":"172.21.15.135:6808","nonce":1099719938}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6804","nonce":1099719938},{"type":"v1","addr":"172.21.15.135:6805","nonce":1099719938}]},"public_addr":"172.21.15.135:6801/1099719938","cluster_addr":"172.21.15.135:6803/1099719938","heartbeat_back_addr":"172.21.15.135:6808/1099719938","heartbeat_front_addr":"172.21.15.135:6805/1099719938","state":["exists","up"]},{"osd":1,"uuid":"fed7a36b-99f5-45e7-8984-de5e09fba654","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6816","nonce":1622923706},{"type":"v1","addr":"172.21.15.135:6817","nonce":1622923706}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6818","nonce":1622923706},{"type":"v1","addr":"172.21.15.135:6819","nonce":1622923706}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6822","nonce":1622923706},{"type":"v1","addr":"172.21.15.135:6823","nonce":1622923706}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6820","nonce":1622923706},{"type":"v1","addr":"172.21.15.135:6821","nonce":1622923706}]},"public_addr":"172.21.15.135:6817/1622923706","cluster_addr":"172.21.15.135:6819/1622923706","heartbeat_back_addr":"172.21.15.135:6823/1622923706","heartbeat_front_addr":"172.21.15.135:6821/1622923706","state":["exists","up"]},{"osd":2,"uuid":"f39a5a76-8dc4-46f2-9e8d-c490677056f6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6807","nonce":1946554517},{"type":"v1","addr":"172.21.15.135:6809","nonce":1946554517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6810","nonce":1946554517},{"type":"v1","addr":"172.21.15.135:6811","nonce":1946554517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6814","nonce":1946554517},{"type":"v1","addr":"172.21.15.135:6815","nonce":1946554517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6812","nonce":1946554517},{"type":"v1","addr":"172.21.15.135:6813","nonce":1946554517}]},"public_addr":"172.21.15.135:6809/1946554517","cluster_addr":"172.21.15.135:6811/1946554517","heartbeat_back_addr":"172.21.15.135:6815/1946554517","heartbeat_front_addr":"172.21.15.135:6813/1946554517","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T08:32:43.433705+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T08:32:43.895674+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T08:32:43.286080+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-14T08:32:55.941 DEBUG:teuthology.orchestra.run.smithi135:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.0 flush_pg_stats 2024-09-14T08:32:55.941 DEBUG:teuthology.orchestra.run.smithi135:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.1 flush_pg_stats 2024-09-14T08:32:55.941 DEBUG:teuthology.orchestra.run.smithi135:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.2 flush_pg_stats 2024-09-14T08:32:56.083 INFO:teuthology.orchestra.run.smithi135.stdout:38654705669 2024-09-14T08:32:56.084 DEBUG:teuthology.orchestra.run.smithi135:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.1 2024-09-14T08:32:56.094 INFO:teuthology.orchestra.run.smithi135.stdout:38654705670 2024-09-14T08:32:56.094 DEBUG:teuthology.orchestra.run.smithi135:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.2 2024-09-14T08:32:56.098 INFO:teuthology.orchestra.run.smithi135.stdout:38654705670 2024-09-14T08:32:56.099 DEBUG:teuthology.orchestra.run.smithi135:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.0 2024-09-14T08:32:56.386 INFO:teuthology.orchestra.run.smithi135.stdout:38654705668 2024-09-14T08:32:56.401 INFO:tasks.ceph.ceph_manager.cluster1:need seq 38654705669 got 38654705668 for osd.1 2024-09-14T08:32:56.483 INFO:teuthology.orchestra.run.smithi135.stdout:38654705669 2024-09-14T08:32:56.488 INFO:teuthology.orchestra.run.smithi135.stdout:38654705669 2024-09-14T08:32:56.497 INFO:tasks.ceph.ceph_manager.cluster1:need seq 38654705670 got 38654705669 for osd.2 2024-09-14T08:32:56.502 INFO:tasks.ceph.ceph_manager.cluster1:need seq 38654705670 got 38654705669 for osd.0 2024-09-14T08:32:57.402 DEBUG:teuthology.orchestra.run.smithi135:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.1 2024-09-14T08:32:57.498 DEBUG:teuthology.orchestra.run.smithi135:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.2 2024-09-14T08:32:57.503 DEBUG:teuthology.orchestra.run.smithi135:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.0 2024-09-14T08:32:57.707 INFO:teuthology.orchestra.run.smithi135.stdout:38654705669 2024-09-14T08:32:57.722 INFO:tasks.ceph.ceph_manager.cluster1:need seq 38654705669 got 38654705669 for osd.1 2024-09-14T08:32:57.722 DEBUG:teuthology.parallel:result is None 2024-09-14T08:32:57.821 INFO:teuthology.orchestra.run.smithi135.stdout:38654705670 2024-09-14T08:32:57.834 INFO:tasks.ceph.ceph_manager.cluster1:need seq 38654705670 got 38654705670 for osd.2 2024-09-14T08:32:57.834 DEBUG:teuthology.parallel:result is None 2024-09-14T08:32:57.861 INFO:teuthology.orchestra.run.smithi135.stdout:38654705670 2024-09-14T08:32:57.874 INFO:tasks.ceph.ceph_manager.cluster1:need seq 38654705670 got 38654705670 for osd.0 2024-09-14T08:32:57.874 DEBUG:teuthology.parallel:result is None 2024-09-14T08:32:57.874 INFO:tasks.ceph.ceph_manager.cluster1:waiting for clean 2024-09-14T08:32:57.874 DEBUG:teuthology.orchestra.run.smithi135:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2024-09-14T08:32:58.196 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:32:58.196 INFO:teuthology.orchestra.run.smithi135.stderr:dumped all 2024-09-14T08:32:58.209 INFO:teuthology.orchestra.run.smithi135.stdout:{"pg_ready":true,"pg_map":{"version":19,"stamp":"2024-09-14T08:32:56.594323+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":81,"num_write_kb":1108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":81640,"kb_used_data":528,"kb_used_omap":4,"kb_used_meta":81019,"kb_avail":283033880,"statfs":{"total":289910292480,"available":289826693120,"internally_reserved":0,"allocated":540672,"data_stored":1301215,"data_compressed":7092,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":4785,"internal_metadata":82963791},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"3.908619"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":20,"reported_epoch":16,"state":"active+clean","last_fresh":"2024-09-14T08:32:54.696491+0000","last_change":"2024-09-14T08:32:54.696724+0000","last_active":"2024-09-14T08:32:54.696491+0000","last_peered":"2024-09-14T08:32:54.696491+0000","last_clean":"2024-09-14T08:32:54.696491+0000","last_became_active":"2024-09-14T08:32:52.693987+0000","last_became_peered":"2024-09-14T08:32:52.693987+0000","last_unstale":"2024-09-14T08:32:54.696491+0000","last_undegraded":"2024-09-14T08:32:54.696491+0000","last_fullsized":"2024-09-14T08:32:54.696491+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:32:51.683516+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:32:51.683516+0000","last_clean_scrub_stamp":"2024-09-14T08:32:51.683516+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T17:10:29.729916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00111144,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.6","version":"0'0","reported_seq":20,"reported_epoch":16,"state":"active+clean","last_fresh":"2024-09-14T08:32:54.696442+0000","last_change":"2024-09-14T08:32:54.696686+0000","last_active":"2024-09-14T08:32:54.696442+0000","last_peered":"2024-09-14T08:32:54.696442+0000","last_clean":"2024-09-14T08:32:54.696442+0000","last_became_active":"2024-09-14T08:32:52.691575+0000","last_became_peered":"2024-09-14T08:32:52.691575+0000","last_unstale":"2024-09-14T08:32:54.696442+0000","last_undegraded":"2024-09-14T08:32:54.696442+0000","last_fullsized":"2024-09-14T08:32:54.696442+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:32:51.683516+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:32:51.683516+0000","last_clean_scrub_stamp":"2024-09-14T08:32:51.683516+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T18:10:27.710040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001078809,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.5","version":"0'0","reported_seq":20,"reported_epoch":16,"state":"active+clean","last_fresh":"2024-09-14T08:32:54.696168+0000","last_change":"2024-09-14T08:32:54.696401+0000","last_active":"2024-09-14T08:32:54.696168+0000","last_peered":"2024-09-14T08:32:54.696168+0000","last_clean":"2024-09-14T08:32:54.696168+0000","last_became_active":"2024-09-14T08:32:52.694100+0000","last_became_peered":"2024-09-14T08:32:52.694100+0000","last_unstale":"2024-09-14T08:32:54.696168+0000","last_undegraded":"2024-09-14T08:32:54.696168+0000","last_fullsized":"2024-09-14T08:32:54.696168+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:32:51.683516+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:32:51.683516+0000","last_clean_scrub_stamp":"2024-09-14T08:32:51.683516+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:48:32.158959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00082548099999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.4","version":"0'0","reported_seq":20,"reported_epoch":16,"state":"active+clean","last_fresh":"2024-09-14T08:32:54.696135+0000","last_change":"2024-09-14T08:32:54.696398+0000","last_active":"2024-09-14T08:32:54.696135+0000","last_peered":"2024-09-14T08:32:54.696135+0000","last_clean":"2024-09-14T08:32:54.696135+0000","last_became_active":"2024-09-14T08:32:52.692903+0000","last_became_peered":"2024-09-14T08:32:52.692903+0000","last_unstale":"2024-09-14T08:32:54.696135+0000","last_undegraded":"2024-09-14T08:32:54.696135+0000","last_fullsized":"2024-09-14T08:32:54.696135+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:32:51.683516+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:32:51.683516+0000","last_clean_scrub_stamp":"2024-09-14T08:32:51.683516+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T20:22:32.946236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00085427300000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.2","version":"16'2","reported_seq":22,"reported_epoch":16,"state":"active+clean","last_fresh":"2024-09-14T08:32:54.692785+0000","last_change":"2024-09-14T08:32:54.692785+0000","last_active":"2024-09-14T08:32:54.692785+0000","last_peered":"2024-09-14T08:32:54.692785+0000","last_clean":"2024-09-14T08:32:54.692785+0000","last_became_active":"2024-09-14T08:32:52.691649+0000","last_became_peered":"2024-09-14T08:32:52.691649+0000","last_unstale":"2024-09-14T08:32:54.692785+0000","last_undegraded":"2024-09-14T08:32:54.692785+0000","last_fullsized":"2024-09-14T08:32:54.692785+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:32:51.683516+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:32:51.683516+0000","last_clean_scrub_stamp":"2024-09-14T08:32:51.683516+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:27:21.636753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00094884699999999995,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.1","version":"0'0","reported_seq":20,"reported_epoch":16,"state":"active+clean","last_fresh":"2024-09-14T08:32:54.689236+0000","last_change":"2024-09-14T08:32:54.689445+0000","last_active":"2024-09-14T08:32:54.689236+0000","last_peered":"2024-09-14T08:32:54.689236+0000","last_clean":"2024-09-14T08:32:54.689236+0000","last_became_active":"2024-09-14T08:32:52.693218+0000","last_became_peered":"2024-09-14T08:32:52.693218+0000","last_unstale":"2024-09-14T08:32:54.689236+0000","last_undegraded":"2024-09-14T08:32:54.689236+0000","last_fullsized":"2024-09-14T08:32:54.689236+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:32:51.683516+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:32:51.683516+0000","last_clean_scrub_stamp":"2024-09-14T08:32:51.683516+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T17:13:15.903145+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000454201,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.0","version":"0'0","reported_seq":20,"reported_epoch":16,"state":"active+clean","last_fresh":"2024-09-14T08:32:54.689168+0000","last_change":"2024-09-14T08:32:54.689408+0000","last_active":"2024-09-14T08:32:54.689168+0000","last_peered":"2024-09-14T08:32:54.689168+0000","last_clean":"2024-09-14T08:32:54.689168+0000","last_became_active":"2024-09-14T08:32:52.693247+0000","last_became_peered":"2024-09-14T08:32:52.693247+0000","last_unstale":"2024-09-14T08:32:54.689168+0000","last_undegraded":"2024-09-14T08:32:54.689168+0000","last_fullsized":"2024-09-14T08:32:54.689168+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:32:51.683516+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:32:51.683516+0000","last_clean_scrub_stamp":"2024-09-14T08:32:51.683516+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:29:38.741547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00044293799999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.3","version":"14'1","reported_seq":21,"reported_epoch":16,"state":"active+clean","last_fresh":"2024-09-14T08:32:54.696804+0000","last_change":"2024-09-14T08:32:54.697190+0000","last_active":"2024-09-14T08:32:54.696804+0000","last_peered":"2024-09-14T08:32:54.696804+0000","last_clean":"2024-09-14T08:32:54.696804+0000","last_became_active":"2024-09-14T08:32:52.691607+0000","last_became_peered":"2024-09-14T08:32:52.691607+0000","last_unstale":"2024-09-14T08:32:54.696804+0000","last_undegraded":"2024-09-14T08:32:54.696804+0000","last_fullsized":"2024-09-14T08:32:54.696804+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:32:51.683516+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:32:51.683516+0000","last_clean_scrub_stamp":"2024-09-14T08:32:51.683516+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T16:28:20.832608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0014408509999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"1.0","version":"12'52","reported_seq":86,"reported_epoch":16,"state":"active+clean","last_fresh":"2024-09-14T08:32:54.695551+0000","last_change":"2024-09-14T08:32:51.096141+0000","last_active":"2024-09-14T08:32:54.695551+0000","last_peered":"2024-09-14T08:32:54.695551+0000","last_clean":"2024-09-14T08:32:54.695551+0000","last_became_active":"2024-09-14T08:32:51.095164+0000","last_became_peered":"2024-09-14T08:32:51.095164+0000","last_unstale":"2024-09-14T08:32:54.695551+0000","last_undegraded":"2024-09-14T08:32:54.695551+0000","last_fullsized":"2024-09-14T08:32:54.695551+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:32:49.668980+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:32:49.668980+0000","last_clean_scrub_stamp":"2024-09-14T08:32:49.668980+0000","objects_scrubbed":0,"log_size":52,"log_dups_size":0,"ondisk_log_size":52,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T19:52:09.318667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":79,"num_write_kb":1106,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":79,"num_write_kb":1106,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":1180736,"data_compressed":7092,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":52,"ondisk_log_size":52,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654705670,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27164,"kb_used_data":136,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344676,"statfs":{"total":96636764160,"available":96608948224,"internally_reserved":0,"allocated":139264,"data_stored":40147,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":9,"seq":38654705669,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27252,"kb_used_data":196,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344588,"statfs":{"total":96636764160,"available":96608858112,"internally_reserved":0,"allocated":200704,"data_stored":630534,"data_compressed":3546,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":9,"seq":38654705670,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27224,"kb_used_data":196,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344616,"statfs":{"total":96636764160,"available":96608886784,"internally_reserved":0,"allocated":200704,"data_stored":630534,"data_compressed":3546,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":3546,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":3546,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T08:32:58.210 DEBUG:teuthology.orchestra.run.smithi135:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2024-09-14T08:32:58.500 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:32:58.501 INFO:teuthology.orchestra.run.smithi135.stderr:dumped all 2024-09-14T08:32:58.513 INFO:teuthology.orchestra.run.smithi135.stdout:{"pg_ready":true,"pg_map":{"version":19,"stamp":"2024-09-14T08:32:56.594323+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":81,"num_write_kb":1108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":81640,"kb_used_data":528,"kb_used_omap":4,"kb_used_meta":81019,"kb_avail":283033880,"statfs":{"total":289910292480,"available":289826693120,"internally_reserved":0,"allocated":540672,"data_stored":1301215,"data_compressed":7092,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":4785,"internal_metadata":82963791},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"3.908619"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":20,"reported_epoch":16,"state":"active+clean","last_fresh":"2024-09-14T08:32:54.696491+0000","last_change":"2024-09-14T08:32:54.696724+0000","last_active":"2024-09-14T08:32:54.696491+0000","last_peered":"2024-09-14T08:32:54.696491+0000","last_clean":"2024-09-14T08:32:54.696491+0000","last_became_active":"2024-09-14T08:32:52.693987+0000","last_became_peered":"2024-09-14T08:32:52.693987+0000","last_unstale":"2024-09-14T08:32:54.696491+0000","last_undegraded":"2024-09-14T08:32:54.696491+0000","last_fullsized":"2024-09-14T08:32:54.696491+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:32:51.683516+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:32:51.683516+0000","last_clean_scrub_stamp":"2024-09-14T08:32:51.683516+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T17:10:29.729916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00111144,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.6","version":"0'0","reported_seq":20,"reported_epoch":16,"state":"active+clean","last_fresh":"2024-09-14T08:32:54.696442+0000","last_change":"2024-09-14T08:32:54.696686+0000","last_active":"2024-09-14T08:32:54.696442+0000","last_peered":"2024-09-14T08:32:54.696442+0000","last_clean":"2024-09-14T08:32:54.696442+0000","last_became_active":"2024-09-14T08:32:52.691575+0000","last_became_peered":"2024-09-14T08:32:52.691575+0000","last_unstale":"2024-09-14T08:32:54.696442+0000","last_undegraded":"2024-09-14T08:32:54.696442+0000","last_fullsized":"2024-09-14T08:32:54.696442+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:32:51.683516+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:32:51.683516+0000","last_clean_scrub_stamp":"2024-09-14T08:32:51.683516+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T18:10:27.710040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001078809,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.5","version":"0'0","reported_seq":20,"reported_epoch":16,"state":"active+clean","last_fresh":"2024-09-14T08:32:54.696168+0000","last_change":"2024-09-14T08:32:54.696401+0000","last_active":"2024-09-14T08:32:54.696168+0000","last_peered":"2024-09-14T08:32:54.696168+0000","last_clean":"2024-09-14T08:32:54.696168+0000","last_became_active":"2024-09-14T08:32:52.694100+0000","last_became_peered":"2024-09-14T08:32:52.694100+0000","last_unstale":"2024-09-14T08:32:54.696168+0000","last_undegraded":"2024-09-14T08:32:54.696168+0000","last_fullsized":"2024-09-14T08:32:54.696168+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:32:51.683516+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:32:51.683516+0000","last_clean_scrub_stamp":"2024-09-14T08:32:51.683516+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:48:32.158959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00082548099999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.4","version":"0'0","reported_seq":20,"reported_epoch":16,"state":"active+clean","last_fresh":"2024-09-14T08:32:54.696135+0000","last_change":"2024-09-14T08:32:54.696398+0000","last_active":"2024-09-14T08:32:54.696135+0000","last_peered":"2024-09-14T08:32:54.696135+0000","last_clean":"2024-09-14T08:32:54.696135+0000","last_became_active":"2024-09-14T08:32:52.692903+0000","last_became_peered":"2024-09-14T08:32:52.692903+0000","last_unstale":"2024-09-14T08:32:54.696135+0000","last_undegraded":"2024-09-14T08:32:54.696135+0000","last_fullsized":"2024-09-14T08:32:54.696135+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:32:51.683516+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:32:51.683516+0000","last_clean_scrub_stamp":"2024-09-14T08:32:51.683516+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T20:22:32.946236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00085427300000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.2","version":"16'2","reported_seq":22,"reported_epoch":16,"state":"active+clean","last_fresh":"2024-09-14T08:32:54.692785+0000","last_change":"2024-09-14T08:32:54.692785+0000","last_active":"2024-09-14T08:32:54.692785+0000","last_peered":"2024-09-14T08:32:54.692785+0000","last_clean":"2024-09-14T08:32:54.692785+0000","last_became_active":"2024-09-14T08:32:52.691649+0000","last_became_peered":"2024-09-14T08:32:52.691649+0000","last_unstale":"2024-09-14T08:32:54.692785+0000","last_undegraded":"2024-09-14T08:32:54.692785+0000","last_fullsized":"2024-09-14T08:32:54.692785+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:32:51.683516+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:32:51.683516+0000","last_clean_scrub_stamp":"2024-09-14T08:32:51.683516+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:27:21.636753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00094884699999999995,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.1","version":"0'0","reported_seq":20,"reported_epoch":16,"state":"active+clean","last_fresh":"2024-09-14T08:32:54.689236+0000","last_change":"2024-09-14T08:32:54.689445+0000","last_active":"2024-09-14T08:32:54.689236+0000","last_peered":"2024-09-14T08:32:54.689236+0000","last_clean":"2024-09-14T08:32:54.689236+0000","last_became_active":"2024-09-14T08:32:52.693218+0000","last_became_peered":"2024-09-14T08:32:52.693218+0000","last_unstale":"2024-09-14T08:32:54.689236+0000","last_undegraded":"2024-09-14T08:32:54.689236+0000","last_fullsized":"2024-09-14T08:32:54.689236+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:32:51.683516+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:32:51.683516+0000","last_clean_scrub_stamp":"2024-09-14T08:32:51.683516+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T17:13:15.903145+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000454201,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.0","version":"0'0","reported_seq":20,"reported_epoch":16,"state":"active+clean","last_fresh":"2024-09-14T08:32:54.689168+0000","last_change":"2024-09-14T08:32:54.689408+0000","last_active":"2024-09-14T08:32:54.689168+0000","last_peered":"2024-09-14T08:32:54.689168+0000","last_clean":"2024-09-14T08:32:54.689168+0000","last_became_active":"2024-09-14T08:32:52.693247+0000","last_became_peered":"2024-09-14T08:32:52.693247+0000","last_unstale":"2024-09-14T08:32:54.689168+0000","last_undegraded":"2024-09-14T08:32:54.689168+0000","last_fullsized":"2024-09-14T08:32:54.689168+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:32:51.683516+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:32:51.683516+0000","last_clean_scrub_stamp":"2024-09-14T08:32:51.683516+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:29:38.741547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00044293799999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.3","version":"14'1","reported_seq":21,"reported_epoch":16,"state":"active+clean","last_fresh":"2024-09-14T08:32:54.696804+0000","last_change":"2024-09-14T08:32:54.697190+0000","last_active":"2024-09-14T08:32:54.696804+0000","last_peered":"2024-09-14T08:32:54.696804+0000","last_clean":"2024-09-14T08:32:54.696804+0000","last_became_active":"2024-09-14T08:32:52.691607+0000","last_became_peered":"2024-09-14T08:32:52.691607+0000","last_unstale":"2024-09-14T08:32:54.696804+0000","last_undegraded":"2024-09-14T08:32:54.696804+0000","last_fullsized":"2024-09-14T08:32:54.696804+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:32:51.683516+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:32:51.683516+0000","last_clean_scrub_stamp":"2024-09-14T08:32:51.683516+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T16:28:20.832608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0014408509999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"1.0","version":"12'52","reported_seq":86,"reported_epoch":16,"state":"active+clean","last_fresh":"2024-09-14T08:32:54.695551+0000","last_change":"2024-09-14T08:32:51.096141+0000","last_active":"2024-09-14T08:32:54.695551+0000","last_peered":"2024-09-14T08:32:54.695551+0000","last_clean":"2024-09-14T08:32:54.695551+0000","last_became_active":"2024-09-14T08:32:51.095164+0000","last_became_peered":"2024-09-14T08:32:51.095164+0000","last_unstale":"2024-09-14T08:32:54.695551+0000","last_undegraded":"2024-09-14T08:32:54.695551+0000","last_fullsized":"2024-09-14T08:32:54.695551+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:32:49.668980+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:32:49.668980+0000","last_clean_scrub_stamp":"2024-09-14T08:32:49.668980+0000","objects_scrubbed":0,"log_size":52,"log_dups_size":0,"ondisk_log_size":52,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T19:52:09.318667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":79,"num_write_kb":1106,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":79,"num_write_kb":1106,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":1180736,"data_compressed":7092,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":52,"ondisk_log_size":52,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654705670,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27164,"kb_used_data":136,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344676,"statfs":{"total":96636764160,"available":96608948224,"internally_reserved":0,"allocated":139264,"data_stored":40147,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":9,"seq":38654705669,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27252,"kb_used_data":196,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344588,"statfs":{"total":96636764160,"available":96608858112,"internally_reserved":0,"allocated":200704,"data_stored":630534,"data_compressed":3546,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":9,"seq":38654705670,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27224,"kb_used_data":196,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344616,"statfs":{"total":96636764160,"available":96608886784,"internally_reserved":0,"allocated":200704,"data_stored":630534,"data_compressed":3546,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":3546,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":3546,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T08:32:58.513 INFO:tasks.ceph.ceph_manager.cluster1:clean! 2024-09-14T08:32:58.514 INFO:tasks.ceph:Waiting until ceph cluster cluster1 is healthy... 2024-09-14T08:32:58.514 INFO:tasks.ceph.ceph_manager.cluster1:wait_until_healthy 2024-09-14T08:32:58.514 DEBUG:teuthology.orchestra.run.smithi135:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 health --format=json 2024-09-14T08:32:58.845 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:32:58.846 INFO:teuthology.orchestra.run.smithi135.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-09-14T08:32:58.859 INFO:tasks.ceph.ceph_manager.cluster1:wait_until_healthy done 2024-09-14T08:32:58.859 INFO:teuthology.run_tasks:Running task ceph... 2024-09-14T08:32:58.868 INFO:tasks.ceph:Creating ceph cluster cluster2... 2024-09-14T08:32:58.868 INFO:tasks.ceph:config {'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'simple'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bluestore block size': 96636764160, 'bluestore compression algorithm': 'zlib', 'bluestore compression mode': 'aggressive', 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'fs': 'xfs', 'mkfs_options': None, 'mount_options': None, 'skip_mgr_daemons': False, 'log_ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'cpu_profile': set(), 'cluster': 'cluster2', 'mon_bind_msgr2': True, 'mon_bind_addrvec': True} 2024-09-14T08:32:58.868 INFO:tasks.ceph:ctx.config {'archive_path': '/home/teuthworker/archive/teuthology-2024-09-10_20:16:17-rbd-main-distro-default-smithi/7900492', 'branch': 'main', 'description': 'rbd/mirror-thrash/{base/install clients/mirror cluster/{2-node openstack} conf/{disable-pool-app} msgr-failures/few objectstore/bluestore-comp-zlib policy/simple rbd-mirror/four-per-cluster supported-random-distro$/{centos_latest} workloads/rbd-mirror-snapshot-stress-workunit-minimum}', 'email': 'ceph-qa@ceph.com', 'first_in_suite': False, 'job_id': '7900492', 'kernel': {'kdb': 1, 'sha1': 'distro'}, 'last_in_suite': False, 'machine_type': 'smithi', 'meta': [{'desc': 'run two ceph clusters and install rbd-mirror'}, {'desc': 'configure the permissions for client.mirror'}, {'desc': '2 ceph clusters with 1 mon and 3 osds each'}, {'desc': 'run four rbd-mirror daemons per cluster'}, {'desc': 'run the rbd_mirror_stress.sh workunit to test the rbd-mirror daemon'}], 'name': 'teuthology-2024-09-10_20:16:17-rbd-main-distro-default-smithi', 'no_nested_subset': False, 'openstack': [{'volumes': {'count': 3, 'size': 30}}], 'os_type': 'centos', 'os_version': '9.stream', 'overrides': {'admin_socket': {'branch': 'main'}, 'ceph': {'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'simple'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bluestore block size': 96636764160, 'bluestore compression algorithm': 'zlib', 'bluestore compression mode': 'aggressive', 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': '7f24e589328ae9e639c0009d93340e5b235d0a47'}, 'ceph-deploy': {'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}}}, 'install': {'ceph': {'flavor': 'default', 'sha1': '7f24e589328ae9e639c0009d93340e5b235d0a47'}}, 'thrashosds': {'bdev_inject_crash': 2, 'bdev_inject_crash_probability': 0.5}, 'workunit': {'branch': 'main', 'sha1': '0d61375b8ab53be4c233f5b93e463edfdff9316a'}}, 'owner': 'scheduled_teuthology@teuthology', 'priority': 950, 'repo': 'https://git.ceph.com/ceph.git', 'roles': [['cluster1.mon.a', 'cluster1.mgr.x', 'cluster2.mgr.x', 'cluster1.osd.0', 'cluster1.osd.1', 'cluster1.osd.2', 'cluster1.client.0', 'cluster2.client.0'], ['cluster2.mon.a', 'cluster2.osd.0', 'cluster2.osd.1', 'cluster2.osd.2', 'cluster1.client.mirror', 'cluster1.client.mirror.0', 'cluster1.client.mirror.1', 'cluster1.client.mirror.2', 'cluster1.client.mirror.3', 'cluster1.client.mirror.4', 'cluster1.client.mirror.5', 'cluster1.client.mirror.6', 'cluster2.client.mirror', 'cluster2.client.mirror.0', 'cluster2.client.mirror.1', 'cluster2.client.mirror.2', 'cluster2.client.mirror.3', 'cluster2.client.mirror.4', 'cluster2.client.mirror.5', 'cluster2.client.mirror.6']], 'seed': 5839, 'sha1': '7f24e589328ae9e639c0009d93340e5b235d0a47', 'sleep_before_teardown': 0, 'subset': '99/128', 'suite': 'rbd', 'suite_branch': 'main', 'suite_path': '/home/teuthworker/src/git.ceph.com_ceph_0d61375b8ab53be4c233f5b93e463edfdff9316a/qa', 'suite_relpath': 'qa', 'suite_repo': 'https://git.ceph.com/ceph.git', 'suite_sha1': '0d61375b8ab53be4c233f5b93e463edfdff9316a', 'targets': {'smithi135.front.sepia.ceph.com': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBJh1m5zJUpHrqEyHhqUVObJhAgM7iP05jGzbnT7mUeekySVDe8iX2BAY4xdaDHj9DekK4x3kUZUrLQKh3KfaYjY=', 'smithi164.front.sepia.ceph.com': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBNmOzx19qAeUG8w7yLVIdwoPWkyQAh2DMyIlga3wbcZH8cBrjHKdqq+xguBx2x/IgM0PT6Rzgxf2HcPAt7MdHzI='}, 'tasks': [{'internal.check_packages': None}, {'internal.buildpackages_prep': None}, {'internal.save_config': None}, {'internal.check_lock': None}, {'internal.add_remotes': None}, {'console_log': None}, {'internal.connect': None}, {'internal.push_inventory': None}, {'internal.serialize_remote_roles': None}, {'internal.check_conflict': None}, {'internal.check_ceph_data': None}, {'internal.vm_setup': None}, {'kernel': {'kdb': 1, 'sha1': 'distro'}}, {'internal.base': None}, {'internal.archive_upload': None}, {'internal.archive': None}, {'internal.coredump': None}, {'internal.sudo': None}, {'internal.syslog': None}, {'internal.timer': None}, {'pcp': None}, {'selinux': None}, {'ansible.cephlab': None}, {'clock': None}, {'install': {'extra_packages': ['rbd-mirror'], 'flavor': 'default', 'sha1': '7f24e589328ae9e639c0009d93340e5b235d0a47'}}, {'ceph': {'cluster': 'cluster1', 'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'simple'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bluestore block size': 96636764160, 'bluestore compression algorithm': 'zlib', 'bluestore compression mode': 'aggressive', 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': '7f24e589328ae9e639c0009d93340e5b235d0a47'}}, {'ceph': {'cluster': 'cluster2', 'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'simple'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bluestore block size': 96636764160, 'bluestore compression algorithm': 'zlib', 'bluestore compression mode': 'aggressive', 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': '7f24e589328ae9e639c0009d93340e5b235d0a47'}}, {'exec': {'cluster1.client.mirror.0': ["sudo ceph --cluster cluster1 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd'"]}}, {'rbd-mirror': {'client': 'cluster1.client.mirror.0', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster1.client.mirror.1', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster1.client.mirror.2', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster1.client.mirror.3', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster2.client.mirror.0', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster2.client.mirror.1', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster2.client.mirror.2', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster2.client.mirror.3', 'thrash': True}}, {'rbd-mirror-thrash': {'cluster': 'cluster1'}}, {'rbd-mirror-thrash': {'cluster': 'cluster2'}}, {'workunit': {'clients': {'cluster1.client.mirror': ['rbd/rbd_mirror_stress.sh']}, 'env': {'CEPH_ARGS': '', 'MIRROR_POOL_MODE': 'image', 'RBD_IMAGE_FEATURES': 'layering', 'RBD_MIRROR_INSTANCES': '4', 'RBD_MIRROR_MODE': 'snapshot', 'RBD_MIRROR_USE_EXISTING_CLUSTER': '1', 'RBD_MIRROR_USE_RBD_MIRROR': '1'}, 'timeout': '6h'}}], 'teuthology': {'fragments_dropped': [], 'meta': {}, 'postmerge': []}, 'teuthology_branch': 'main', 'teuthology_sha1': '3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a', 'timestamp': '2024-09-10_20:16:17', 'tube': 'smithi', 'user': 'teuthology', 'verbose': False, 'worker_log': '/home/teuthworker/archive/worker_logs/dispatcher.smithi.3401856'} 2024-09-14T08:32:58.869 DEBUG:teuthology.orchestra.run.smithi135:> install -d -m0755 -- /home/ubuntu/cephtest/cluster2.data 2024-09-14T08:32:58.871 DEBUG:teuthology.orchestra.run.smithi164:> install -d -m0755 -- /home/ubuntu/cephtest/cluster2.data 2024-09-14T08:32:58.890 DEBUG:teuthology.orchestra.run.smithi135:> sudo install -d -m0777 -- /var/run/ceph 2024-09-14T08:32:58.926 DEBUG:teuthology.orchestra.run.smithi164:> sudo install -d -m0777 -- /var/run/ceph 2024-09-14T08:32:58.959 DEBUG:teuthology.orchestra.run.smithi164:> set -ex 2024-09-14T08:32:58.960 DEBUG:teuthology.orchestra.run.smithi164:> dd if=/scratch_devs of=/dev/stdout 2024-09-14T08:32:59.018 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-09-14T08:32:59.018 DEBUG:teuthology.orchestra.run.smithi164:> stat /dev/vg_nvme/lv_1 2024-09-14T08:32:59.076 INFO:teuthology.orchestra.run.smithi164.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-09-14T08:32:59.076 INFO:teuthology.orchestra.run.smithi164.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-14T08:32:59.076 INFO:teuthology.orchestra.run.smithi164.stdout:Device: 5h/5d Inode: 668 Links: 1 2024-09-14T08:32:59.076 INFO:teuthology.orchestra.run.smithi164.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-14T08:32:59.076 INFO:teuthology.orchestra.run.smithi164.stdout:Context: system_u:object_r:device_t:s0 2024-09-14T08:32:59.076 INFO:teuthology.orchestra.run.smithi164.stdout:Access: 2024-09-14 08:32:14.584776639 +0000 2024-09-14T08:32:59.076 INFO:teuthology.orchestra.run.smithi164.stdout:Modify: 2024-09-14 08:29:38.446373878 +0000 2024-09-14T08:32:59.076 INFO:teuthology.orchestra.run.smithi164.stdout:Change: 2024-09-14 08:29:38.446373878 +0000 2024-09-14T08:32:59.076 INFO:teuthology.orchestra.run.smithi164.stdout: Birth: 2024-09-14 08:29:38.446373878 +0000 2024-09-14T08:32:59.077 DEBUG:teuthology.orchestra.run.smithi164:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-09-14T08:32:59.140 INFO:teuthology.orchestra.run.smithi164.stderr:1+0 records in 2024-09-14T08:32:59.140 INFO:teuthology.orchestra.run.smithi164.stderr:1+0 records out 2024-09-14T08:32:59.140 INFO:teuthology.orchestra.run.smithi164.stderr:512 bytes copied, 0.000205611 s, 2.5 MB/s 2024-09-14T08:32:59.141 DEBUG:teuthology.orchestra.run.smithi164:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-09-14T08:32:59.197 DEBUG:teuthology.orchestra.run.smithi164:> stat /dev/vg_nvme/lv_2 2024-09-14T08:32:59.253 INFO:teuthology.orchestra.run.smithi164.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-09-14T08:32:59.254 INFO:teuthology.orchestra.run.smithi164.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-14T08:32:59.254 INFO:teuthology.orchestra.run.smithi164.stdout:Device: 5h/5d Inode: 685 Links: 1 2024-09-14T08:32:59.254 INFO:teuthology.orchestra.run.smithi164.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-14T08:32:59.254 INFO:teuthology.orchestra.run.smithi164.stdout:Context: system_u:object_r:device_t:s0 2024-09-14T08:32:59.254 INFO:teuthology.orchestra.run.smithi164.stdout:Access: 2024-09-14 08:32:14.584776639 +0000 2024-09-14T08:32:59.254 INFO:teuthology.orchestra.run.smithi164.stdout:Modify: 2024-09-14 08:29:38.767368991 +0000 2024-09-14T08:32:59.254 INFO:teuthology.orchestra.run.smithi164.stdout:Change: 2024-09-14 08:29:38.767368991 +0000 2024-09-14T08:32:59.254 INFO:teuthology.orchestra.run.smithi164.stdout: Birth: 2024-09-14 08:29:38.767368991 +0000 2024-09-14T08:32:59.254 DEBUG:teuthology.orchestra.run.smithi164:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-09-14T08:32:59.317 INFO:teuthology.orchestra.run.smithi164.stderr:1+0 records in 2024-09-14T08:32:59.318 INFO:teuthology.orchestra.run.smithi164.stderr:1+0 records out 2024-09-14T08:32:59.318 INFO:teuthology.orchestra.run.smithi164.stderr:512 bytes copied, 0.000201478 s, 2.5 MB/s 2024-09-14T08:32:59.319 DEBUG:teuthology.orchestra.run.smithi164:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-09-14T08:32:59.375 DEBUG:teuthology.orchestra.run.smithi164:> stat /dev/vg_nvme/lv_3 2024-09-14T08:32:59.432 INFO:teuthology.orchestra.run.smithi164.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-09-14T08:32:59.432 INFO:teuthology.orchestra.run.smithi164.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-14T08:32:59.432 INFO:teuthology.orchestra.run.smithi164.stdout:Device: 5h/5d Inode: 705 Links: 1 2024-09-14T08:32:59.432 INFO:teuthology.orchestra.run.smithi164.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-14T08:32:59.432 INFO:teuthology.orchestra.run.smithi164.stdout:Context: system_u:object_r:device_t:s0 2024-09-14T08:32:59.433 INFO:teuthology.orchestra.run.smithi164.stdout:Access: 2024-09-14 08:32:14.584776639 +0000 2024-09-14T08:32:59.433 INFO:teuthology.orchestra.run.smithi164.stdout:Modify: 2024-09-14 08:29:39.091364059 +0000 2024-09-14T08:32:59.433 INFO:teuthology.orchestra.run.smithi164.stdout:Change: 2024-09-14 08:29:39.091364059 +0000 2024-09-14T08:32:59.433 INFO:teuthology.orchestra.run.smithi164.stdout: Birth: 2024-09-14 08:29:39.091364059 +0000 2024-09-14T08:32:59.433 DEBUG:teuthology.orchestra.run.smithi164:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-09-14T08:32:59.496 INFO:teuthology.orchestra.run.smithi164.stderr:1+0 records in 2024-09-14T08:32:59.496 INFO:teuthology.orchestra.run.smithi164.stderr:1+0 records out 2024-09-14T08:32:59.496 INFO:teuthology.orchestra.run.smithi164.stderr:512 bytes copied, 0.000177456 s, 2.9 MB/s 2024-09-14T08:32:59.497 DEBUG:teuthology.orchestra.run.smithi164:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-09-14T08:32:59.553 DEBUG:teuthology.orchestra.run.smithi164:> stat /dev/vg_nvme/lv_4 2024-09-14T08:32:59.608 INFO:teuthology.orchestra.run.smithi164.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-09-14T08:32:59.609 INFO:teuthology.orchestra.run.smithi164.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-14T08:32:59.609 INFO:teuthology.orchestra.run.smithi164.stdout:Device: 5h/5d Inode: 722 Links: 1 2024-09-14T08:32:59.609 INFO:teuthology.orchestra.run.smithi164.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-14T08:32:59.609 INFO:teuthology.orchestra.run.smithi164.stdout:Context: system_u:object_r:device_t:s0 2024-09-14T08:32:59.609 INFO:teuthology.orchestra.run.smithi164.stdout:Access: 2024-09-14 08:32:14.585776623 +0000 2024-09-14T08:32:59.609 INFO:teuthology.orchestra.run.smithi164.stdout:Modify: 2024-09-14 08:29:39.407359249 +0000 2024-09-14T08:32:59.609 INFO:teuthology.orchestra.run.smithi164.stdout:Change: 2024-09-14 08:29:39.407359249 +0000 2024-09-14T08:32:59.609 INFO:teuthology.orchestra.run.smithi164.stdout: Birth: 2024-09-14 08:29:39.407359249 +0000 2024-09-14T08:32:59.609 DEBUG:teuthology.orchestra.run.smithi164:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-09-14T08:32:59.672 INFO:teuthology.orchestra.run.smithi164.stderr:1+0 records in 2024-09-14T08:32:59.672 INFO:teuthology.orchestra.run.smithi164.stderr:1+0 records out 2024-09-14T08:32:59.672 INFO:teuthology.orchestra.run.smithi164.stderr:512 bytes copied, 0.00016687 s, 3.1 MB/s 2024-09-14T08:32:59.673 DEBUG:teuthology.orchestra.run.smithi164:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-09-14T08:32:59.729 INFO:tasks.ceph:osd dev map: {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'} 2024-09-14T08:32:59.729 INFO:tasks.ceph:remote_to_roles_to_devs: {Remote(name='ubuntu@smithi164.front.sepia.ceph.com'): {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'}} 2024-09-14T08:32:59.730 INFO:tasks.ceph:Generating config... 2024-09-14T08:32:59.731 INFO:tasks.ceph:[client] debug rbd = 20 2024-09-14T08:32:59.731 INFO:tasks.ceph:[client] debug rbd_mirror = 15 2024-09-14T08:32:59.731 INFO:tasks.ceph:[client] log to stderr = False 2024-09-14T08:32:59.731 INFO:tasks.ceph:[client] rbd default features = 125 2024-09-14T08:32:59.731 INFO:tasks.ceph:[client] rbd mirror image policy type = simple 2024-09-14T08:32:59.731 INFO:tasks.ceph:[client.mirror.0] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-14T08:32:59.731 INFO:tasks.ceph:[client.mirror.0] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-14T08:32:59.731 INFO:tasks.ceph:[client.mirror.1] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-14T08:32:59.731 INFO:tasks.ceph:[client.mirror.1] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-14T08:32:59.731 INFO:tasks.ceph:[client.mirror.2] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-14T08:32:59.731 INFO:tasks.ceph:[client.mirror.2] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-14T08:32:59.731 INFO:tasks.ceph:[client.mirror.3] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-14T08:32:59.732 INFO:tasks.ceph:[client.mirror.3] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-14T08:32:59.732 INFO:tasks.ceph:[global] mon client directed command retry = 5 2024-09-14T08:32:59.732 INFO:tasks.ceph:[global] mon warn on pool no app = False 2024-09-14T08:32:59.732 INFO:tasks.ceph:[global] ms inject socket failures = 5000 2024-09-14T08:32:59.732 INFO:tasks.ceph:[mgr] debug mgr = 20 2024-09-14T08:32:59.732 INFO:tasks.ceph:[mgr] debug ms = 1 2024-09-14T08:32:59.732 INFO:tasks.ceph:[mon] debug mon = 20 2024-09-14T08:32:59.732 INFO:tasks.ceph:[mon] debug ms = 1 2024-09-14T08:32:59.732 INFO:tasks.ceph:[mon] debug paxos = 20 2024-09-14T08:32:59.732 INFO:tasks.ceph:[osd] bluestore block size = 96636764160 2024-09-14T08:32:59.732 INFO:tasks.ceph:[osd] bluestore compression algorithm = zlib 2024-09-14T08:32:59.732 INFO:tasks.ceph:[osd] bluestore compression mode = aggressive 2024-09-14T08:32:59.732 INFO:tasks.ceph:[osd] bluestore fsck on mount = True 2024-09-14T08:32:59.733 INFO:tasks.ceph:[osd] debug bluefs = 1/20 2024-09-14T08:32:59.733 INFO:tasks.ceph:[osd] debug bluestore = 1/20 2024-09-14T08:32:59.733 INFO:tasks.ceph:[osd] debug ms = 1 2024-09-14T08:32:59.733 INFO:tasks.ceph:[osd] debug osd = 20 2024-09-14T08:32:59.733 INFO:tasks.ceph:[osd] debug rocksdb = 4/10 2024-09-14T08:32:59.733 INFO:tasks.ceph:[osd] mon osd backfillfull_ratio = 0.85 2024-09-14T08:32:59.733 INFO:tasks.ceph:[osd] mon osd full ratio = 0.9 2024-09-14T08:32:59.733 INFO:tasks.ceph:[osd] mon osd nearfull ratio = 0.8 2024-09-14T08:32:59.733 INFO:tasks.ceph:[osd] osd failsafe full ratio = 0.95 2024-09-14T08:32:59.733 INFO:tasks.ceph:[osd] osd objectstore = bluestore 2024-09-14T08:32:59.733 INFO:tasks.ceph:Setting up cluster2.mon.a... 2024-09-14T08:32:59.733 DEBUG:teuthology.orchestra.run.smithi164:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring /etc/ceph/cluster2.keyring 2024-09-14T08:32:59.808 INFO:teuthology.orchestra.run.smithi164.stdout:creating /etc/ceph/cluster2.keyring 2024-09-14T08:32:59.811 DEBUG:teuthology.orchestra.run.smithi164:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=mon. /etc/ceph/cluster2.keyring 2024-09-14T08:32:59.894 DEBUG:teuthology.orchestra.run.smithi164:> sudo chmod 0644 /etc/ceph/cluster2.keyring 2024-09-14T08:32:59.958 DEBUG:tasks.ceph:Ceph mon addresses: [('cluster2.mon.a', '172.21.15.164')] 2024-09-14T08:32:59.958 DEBUG:tasks.ceph:writing out conf {'global': {'chdir': '', 'pid file': '/var/run/ceph/$cluster-$name.pid', 'auth supported': 'cephx', 'filestore xattr use omap': 'true', 'mon clock drift allowed': '1.000', 'osd crush chooseleaf type': '0', 'auth debug': 'true', 'ms die on old message': 'true', 'ms die on bug': 'true', 'mon max pg per osd': '10000', 'mon pg warn max object skew': '0', 'osd_pool_default_pg_autoscale_mode': 'off', 'osd pool default size': '2', 'mon osd allow primary affinity': 'true', 'mon osd allow pg remap': 'true', 'mon warn on legacy crush tunables': 'false', 'mon warn on crush straw calc version zero': 'false', 'mon warn on no sortbitwise': 'false', 'mon warn on osd down out interval zero': 'false', 'mon warn on too few osds': 'false', 'mon_warn_on_pool_pg_num_not_power_of_two': 'false', 'mon_warn_on_pool_no_redundancy': 'false', 'mon_allow_pool_size_one': 'true', 'osd pool default erasure code profile': 'plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd', 'osd default data pool replay window': '5', 'mon allow pool delete': 'true', 'mon cluster log file level': 'debug', 'debug asserts on shutdown': 'true', 'mon health detail to clog': 'false', 'mon host': '172.21.15.164', 'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'osd': {'osd journal size': '100', 'osd scrub load threshold': '5.0', 'osd scrub max interval': '600', 'osd mclock profile': 'high_recovery_ops', 'osd recover clone overlap': 'true', 'osd recovery max chunk': '1048576', 'osd debug shutdown': 'true', 'osd debug op order': 'true', 'osd debug verify stray on activate': 'true', 'osd debug trim objects': 'true', 'osd open classes on start': 'true', 'osd debug pg log writeout': 'true', 'osd deep scrub update digest min age': '30', 'osd map max advance': '10', 'journal zero on create': 'true', 'filestore ondisk finisher threads': '3', 'filestore apply finisher threads': '3', 'bdev debug aio': 'true', 'osd debug misdirected ops': 'true', 'bluestore block size': 96636764160, 'bluestore compression algorithm': 'zlib', 'bluestore compression mode': 'aggressive', 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}, 'mgr': {'debug ms': 1, 'debug mgr': 20, 'debug mon': '20', 'debug auth': '20', 'mon reweight min pgs per osd': '4', 'mon reweight min bytes per osd': '10', 'mgr/telemetry/nag': 'false'}, 'mon': {'debug ms': 1, 'debug mon': 20, 'debug paxos': 20, 'debug auth': '20', 'mon data avail warn': '5', 'mon mgr mkfs grace': '240', 'mon reweight min pgs per osd': '4', 'mon osd reporter subtree level': 'osd', 'mon osd prime pg temp': 'true', 'mon reweight min bytes per osd': '10', 'auth mon ticket ttl': '660', 'auth service ticket ttl': '240', 'mon_warn_on_insecure_global_id_reclaim': 'false', 'mon_warn_on_insecure_global_id_reclaim_allowed': 'false', 'mon_down_mkfs_grace': '2m', 'mon_warn_on_filestore_osds': 'false'}, 'client': {'rgw cache enabled': 'true', 'rgw enable ops log': 'true', 'rgw enable usage log': 'true', 'log file': '/var/log/ceph/$cluster-$name.$pid.log', 'admin socket': '/var/run/ceph/$cluster-$name.$pid.asok', 'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'simple'}, 'mon.a': {}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}} 2024-09-14T08:32:59.959 DEBUG:teuthology.orchestra.run.smithi164:> set -ex 2024-09-14T08:32:59.959 DEBUG:teuthology.orchestra.run.smithi164:> dd of=/home/ubuntu/cephtest/ceph.tmp.conf 2024-09-14T08:33:00.013 DEBUG:teuthology.orchestra.run.smithi164:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage monmaptool -c /home/ubuntu/cephtest/ceph.tmp.conf --create --clobber --enable-all-features --add a 172.21.15.164 --print /home/ubuntu/cephtest/cluster2.monmap 2024-09-14T08:33:00.093 INFO:teuthology.orchestra.run.smithi164.stdout:monmaptool: monmap file /home/ubuntu/cephtest/cluster2.monmap 2024-09-14T08:33:00.093 INFO:teuthology.orchestra.run.smithi164.stdout:monmaptool: generated fsid dd8fb0de-ee38-4627-a696-b631746d355c 2024-09-14T08:33:00.094 INFO:teuthology.orchestra.run.smithi164.stdout:setting min_mon_release = quincy 2024-09-14T08:33:00.094 INFO:teuthology.orchestra.run.smithi164.stdout:epoch 0 2024-09-14T08:33:00.094 INFO:teuthology.orchestra.run.smithi164.stdout:fsid dd8fb0de-ee38-4627-a696-b631746d355c 2024-09-14T08:33:00.094 INFO:teuthology.orchestra.run.smithi164.stdout:last_changed 2024-09-14T08:33:00.094918+0000 2024-09-14T08:33:00.094 INFO:teuthology.orchestra.run.smithi164.stdout:created 2024-09-14T08:33:00.094918+0000 2024-09-14T08:33:00.094 INFO:teuthology.orchestra.run.smithi164.stdout:min_mon_release 17 (quincy) 2024-09-14T08:33:00.094 INFO:teuthology.orchestra.run.smithi164.stdout:election_strategy: 1 2024-09-14T08:33:00.094 INFO:teuthology.orchestra.run.smithi164.stdout:0: [v2:172.21.15.164:3300/0,v1:172.21.15.164:6789/0] mon.a 2024-09-14T08:33:00.094 INFO:teuthology.orchestra.run.smithi164.stdout:monmaptool: writing epoch 0 to /home/ubuntu/cephtest/cluster2.monmap (1 monitors) 2024-09-14T08:33:00.095 DEBUG:teuthology.orchestra.run.smithi164:> rm -- /home/ubuntu/cephtest/ceph.tmp.conf 2024-09-14T08:33:00.150 INFO:tasks.ceph:Writing /etc/ceph/cluster2.conf for FSID dd8fb0de-ee38-4627-a696-b631746d355c... 2024-09-14T08:33:00.151 DEBUG:teuthology.orchestra.run.smithi135:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/cluster2.conf && sudo chmod 0644 /etc/ceph/cluster2.conf > /dev/null 2024-09-14T08:33:00.154 DEBUG:teuthology.orchestra.run.smithi164:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/cluster2.conf && sudo chmod 0644 /etc/ceph/cluster2.conf > /dev/null 2024-09-14T08:33:00.199 INFO:teuthology.orchestra.run.smithi135.stdout:[global] 2024-09-14T08:33:00.199 INFO:teuthology.orchestra.run.smithi135.stdout: chdir = "" 2024-09-14T08:33:00.200 INFO:teuthology.orchestra.run.smithi135.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2024-09-14T08:33:00.200 INFO:teuthology.orchestra.run.smithi135.stdout: auth supported = cephx 2024-09-14T08:33:00.200 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:33:00.200 INFO:teuthology.orchestra.run.smithi135.stdout: filestore xattr use omap = true 2024-09-14T08:33:00.200 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:33:00.200 INFO:teuthology.orchestra.run.smithi135.stdout: mon clock drift allowed = 1.000 2024-09-14T08:33:00.200 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:33:00.200 INFO:teuthology.orchestra.run.smithi135.stdout: osd crush chooseleaf type = 0 2024-09-14T08:33:00.200 INFO:teuthology.orchestra.run.smithi135.stdout: auth debug = true 2024-09-14T08:33:00.200 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:33:00.200 INFO:teuthology.orchestra.run.smithi135.stdout: ms die on old message = true 2024-09-14T08:33:00.200 INFO:teuthology.orchestra.run.smithi135.stdout: ms die on bug = true 2024-09-14T08:33:00.200 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:33:00.201 INFO:teuthology.orchestra.run.smithi135.stdout: mon max pg per osd = 10000 # >= luminous 2024-09-14T08:33:00.201 INFO:teuthology.orchestra.run.smithi135.stdout: mon pg warn max object skew = 0 2024-09-14T08:33:00.201 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:33:00.201 INFO:teuthology.orchestra.run.smithi135.stdout: # disable pg_autoscaler by default for new pools 2024-09-14T08:33:00.201 INFO:teuthology.orchestra.run.smithi135.stdout: osd_pool_default_pg_autoscale_mode = off 2024-09-14T08:33:00.201 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:33:00.201 INFO:teuthology.orchestra.run.smithi135.stdout: osd pool default size = 2 2024-09-14T08:33:00.201 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:33:00.201 INFO:teuthology.orchestra.run.smithi135.stdout: mon osd allow primary affinity = true 2024-09-14T08:33:00.201 INFO:teuthology.orchestra.run.smithi135.stdout: mon osd allow pg remap = true 2024-09-14T08:33:00.201 INFO:teuthology.orchestra.run.smithi135.stdout: mon warn on legacy crush tunables = false 2024-09-14T08:33:00.201 INFO:teuthology.orchestra.run.smithi135.stdout: mon warn on crush straw calc version zero = false 2024-09-14T08:33:00.201 INFO:teuthology.orchestra.run.smithi135.stdout: mon warn on no sortbitwise = false 2024-09-14T08:33:00.202 INFO:teuthology.orchestra.run.smithi135.stdout: mon warn on osd down out interval zero = false 2024-09-14T08:33:00.202 INFO:teuthology.orchestra.run.smithi135.stdout: mon warn on too few osds = false 2024-09-14T08:33:00.202 INFO:teuthology.orchestra.run.smithi135.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2024-09-14T08:33:00.202 INFO:teuthology.orchestra.run.smithi135.stdout: mon_warn_on_pool_no_redundancy = false 2024-09-14T08:33:00.202 INFO:teuthology.orchestra.run.smithi135.stdout: mon_allow_pool_size_one = true 2024-09-14T08:33:00.202 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:33:00.202 INFO:teuthology.orchestra.run.smithi135.stdout: osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2024-09-14T08:33:00.202 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:33:00.202 INFO:teuthology.orchestra.run.smithi135.stdout: osd default data pool replay window = 5 2024-09-14T08:33:00.202 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:33:00.202 INFO:teuthology.orchestra.run.smithi135.stdout: mon allow pool delete = true 2024-09-14T08:33:00.202 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:33:00.202 INFO:teuthology.orchestra.run.smithi135.stdout: mon cluster log file level = debug 2024-09-14T08:33:00.202 INFO:teuthology.orchestra.run.smithi135.stdout: debug asserts on shutdown = true 2024-09-14T08:33:00.203 INFO:teuthology.orchestra.run.smithi135.stdout: mon health detail to clog = false 2024-09-14T08:33:00.203 INFO:teuthology.orchestra.run.smithi135.stdout: mon host = 172.21.15.164 2024-09-14T08:33:00.203 INFO:teuthology.orchestra.run.smithi135.stdout: mon client directed command retry = 5 2024-09-14T08:33:00.203 INFO:teuthology.orchestra.run.smithi135.stdout: mon warn on pool no app = False 2024-09-14T08:33:00.203 INFO:teuthology.orchestra.run.smithi135.stdout: ms inject socket failures = 5000 2024-09-14T08:33:00.203 INFO:teuthology.orchestra.run.smithi135.stdout: fsid = dd8fb0de-ee38-4627-a696-b631746d355c 2024-09-14T08:33:00.203 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:33:00.203 INFO:teuthology.orchestra.run.smithi135.stdout:[osd] 2024-09-14T08:33:00.203 INFO:teuthology.orchestra.run.smithi135.stdout: osd journal size = 100 2024-09-14T08:33:00.203 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:33:00.203 INFO:teuthology.orchestra.run.smithi135.stdout: osd scrub load threshold = 5.0 2024-09-14T08:33:00.203 INFO:teuthology.orchestra.run.smithi135.stdout: osd scrub max interval = 600 2024-09-14T08:33:00.203 INFO:teuthology.orchestra.run.smithi135.stdout: osd mclock profile = high_recovery_ops 2024-09-14T08:33:00.203 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:33:00.203 INFO:teuthology.orchestra.run.smithi135.stdout: osd recover clone overlap = true 2024-09-14T08:33:00.204 INFO:teuthology.orchestra.run.smithi135.stdout: osd recovery max chunk = 1048576 2024-09-14T08:33:00.204 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:33:00.204 INFO:teuthology.orchestra.run.smithi135.stdout: osd debug shutdown = true 2024-09-14T08:33:00.204 INFO:teuthology.orchestra.run.smithi135.stdout: osd debug op order = true 2024-09-14T08:33:00.204 INFO:teuthology.orchestra.run.smithi135.stdout: osd debug verify stray on activate = true 2024-09-14T08:33:00.204 INFO:teuthology.orchestra.run.smithi135.stdout: osd debug trim objects = true 2024-09-14T08:33:00.204 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:33:00.204 INFO:teuthology.orchestra.run.smithi135.stdout: osd open classes on start = true 2024-09-14T08:33:00.204 INFO:teuthology.orchestra.run.smithi135.stdout: osd debug pg log writeout = true 2024-09-14T08:33:00.204 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:33:00.204 INFO:teuthology.orchestra.run.smithi135.stdout: osd deep scrub update digest min age = 30 2024-09-14T08:33:00.204 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:33:00.204 INFO:teuthology.orchestra.run.smithi135.stdout: osd map max advance = 10 2024-09-14T08:33:00.204 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:33:00.205 INFO:teuthology.orchestra.run.smithi135.stdout: journal zero on create = true 2024-09-14T08:33:00.205 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:33:00.205 INFO:teuthology.orchestra.run.smithi135.stdout: filestore ondisk finisher threads = 3 2024-09-14T08:33:00.205 INFO:teuthology.orchestra.run.smithi135.stdout: filestore apply finisher threads = 3 2024-09-14T08:33:00.205 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:33:00.205 INFO:teuthology.orchestra.run.smithi135.stdout: bdev debug aio = true 2024-09-14T08:33:00.205 INFO:teuthology.orchestra.run.smithi135.stdout: osd debug misdirected ops = true 2024-09-14T08:33:00.205 INFO:teuthology.orchestra.run.smithi135.stdout: bluestore block size = 96636764160 2024-09-14T08:33:00.205 INFO:teuthology.orchestra.run.smithi135.stdout: bluestore compression algorithm = zlib 2024-09-14T08:33:00.205 INFO:teuthology.orchestra.run.smithi135.stdout: bluestore compression mode = aggressive 2024-09-14T08:33:00.205 INFO:teuthology.orchestra.run.smithi135.stdout: bluestore fsck on mount = True 2024-09-14T08:33:00.205 INFO:teuthology.orchestra.run.smithi135.stdout: debug bluefs = 1/20 2024-09-14T08:33:00.205 INFO:teuthology.orchestra.run.smithi135.stdout: debug bluestore = 1/20 2024-09-14T08:33:00.205 INFO:teuthology.orchestra.run.smithi135.stdout: debug ms = 1 2024-09-14T08:33:00.206 INFO:teuthology.orchestra.run.smithi135.stdout: debug osd = 20 2024-09-14T08:33:00.206 INFO:teuthology.orchestra.run.smithi135.stdout: debug rocksdb = 4/10 2024-09-14T08:33:00.206 INFO:teuthology.orchestra.run.smithi135.stdout: mon osd backfillfull_ratio = 0.85 2024-09-14T08:33:00.206 INFO:teuthology.orchestra.run.smithi135.stdout: mon osd full ratio = 0.9 2024-09-14T08:33:00.206 INFO:teuthology.orchestra.run.smithi135.stdout: mon osd nearfull ratio = 0.8 2024-09-14T08:33:00.206 INFO:teuthology.orchestra.run.smithi135.stdout: osd failsafe full ratio = 0.95 2024-09-14T08:33:00.206 INFO:teuthology.orchestra.run.smithi135.stdout: osd objectstore = bluestore 2024-09-14T08:33:00.206 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:33:00.206 INFO:teuthology.orchestra.run.smithi135.stdout:[mgr] 2024-09-14T08:33:00.206 INFO:teuthology.orchestra.run.smithi135.stdout: debug ms = 1 2024-09-14T08:33:00.206 INFO:teuthology.orchestra.run.smithi135.stdout: debug mgr = 20 2024-09-14T08:33:00.206 INFO:teuthology.orchestra.run.smithi135.stdout: debug mon = 20 2024-09-14T08:33:00.207 INFO:teuthology.orchestra.run.smithi135.stdout: debug auth = 20 2024-09-14T08:33:00.207 INFO:teuthology.orchestra.run.smithi135.stdout: mon reweight min pgs per osd = 4 2024-09-14T08:33:00.207 INFO:teuthology.orchestra.run.smithi135.stdout: mon reweight min bytes per osd = 10 2024-09-14T08:33:00.207 INFO:teuthology.orchestra.run.smithi135.stdout: mgr/telemetry/nag = false 2024-09-14T08:33:00.207 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:33:00.207 INFO:teuthology.orchestra.run.smithi135.stdout:[mon] 2024-09-14T08:33:00.207 INFO:teuthology.orchestra.run.smithi135.stdout: debug ms = 1 2024-09-14T08:33:00.207 INFO:teuthology.orchestra.run.smithi135.stdout: debug mon = 20 2024-09-14T08:33:00.207 INFO:teuthology.orchestra.run.smithi135.stdout: debug paxos = 20 2024-09-14T08:33:00.207 INFO:teuthology.orchestra.run.smithi135.stdout: debug auth = 20 2024-09-14T08:33:00.207 INFO:teuthology.orchestra.run.smithi135.stdout: mon data avail warn = 5 2024-09-14T08:33:00.208 INFO:teuthology.orchestra.run.smithi135.stdout: mon mgr mkfs grace = 240 2024-09-14T08:33:00.208 INFO:teuthology.orchestra.run.smithi135.stdout: mon reweight min pgs per osd = 4 2024-09-14T08:33:00.208 INFO:teuthology.orchestra.run.smithi135.stdout: mon osd reporter subtree level = osd 2024-09-14T08:33:00.208 INFO:teuthology.orchestra.run.smithi135.stdout: mon osd prime pg temp = true 2024-09-14T08:33:00.208 INFO:teuthology.orchestra.run.smithi135.stdout: mon reweight min bytes per osd = 10 2024-09-14T08:33:00.208 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:33:00.208 INFO:teuthology.orchestra.run.smithi135.stdout: # rotate auth tickets quickly to exercise renewal paths 2024-09-14T08:33:00.208 INFO:teuthology.orchestra.run.smithi135.stdout: auth mon ticket ttl = 660 # 11m 2024-09-14T08:33:00.208 INFO:teuthology.orchestra.run.smithi135.stdout: auth service ticket ttl = 240 # 4m 2024-09-14T08:33:00.208 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:33:00.208 INFO:teuthology.orchestra.run.smithi135.stdout: # don't complain about insecure global_id in the test suite 2024-09-14T08:33:00.208 INFO:teuthology.orchestra.run.smithi135.stdout: mon_warn_on_insecure_global_id_reclaim = false 2024-09-14T08:33:00.208 INFO:teuthology.orchestra.run.smithi135.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2024-09-14T08:33:00.209 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:33:00.209 INFO:teuthology.orchestra.run.smithi135.stdout: # 1m isn't quite enough 2024-09-14T08:33:00.209 INFO:teuthology.orchestra.run.smithi135.stdout: mon_down_mkfs_grace = 2m 2024-09-14T08:33:00.209 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:33:00.209 INFO:teuthology.orchestra.run.smithi135.stdout: mon_warn_on_filestore_osds = false 2024-09-14T08:33:00.209 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T08:33:00.209 INFO:teuthology.orchestra.run.smithi135.stdout:[client] 2024-09-14T08:33:00.209 INFO:teuthology.orchestra.run.smithi135.stdout: rgw cache enabled = true 2024-09-14T08:33:00.209 INFO:teuthology.orchestra.run.smithi135.stdout: rgw enable ops log = true 2024-09-14T08:33:00.209 INFO:teuthology.orchestra.run.smithi135.stdout: rgw enable usage log = true 2024-09-14T08:33:00.209 INFO:teuthology.orchestra.run.smithi135.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2024-09-14T08:33:00.209 INFO:teuthology.orchestra.run.smithi135.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2024-09-14T08:33:00.209 INFO:teuthology.orchestra.run.smithi135.stdout: debug rbd = 20 2024-09-14T08:33:00.210 INFO:teuthology.orchestra.run.smithi135.stdout: debug rbd_mirror = 15 2024-09-14T08:33:00.210 INFO:teuthology.orchestra.run.smithi135.stdout: log to stderr = False 2024-09-14T08:33:00.210 INFO:teuthology.orchestra.run.smithi135.stdout: rbd default features = 125 2024-09-14T08:33:00.210 INFO:teuthology.orchestra.run.smithi135.stdout: rbd mirror image policy type = simple 2024-09-14T08:33:00.210 INFO:teuthology.orchestra.run.smithi135.stdout:[mon.a] 2024-09-14T08:33:00.210 INFO:teuthology.orchestra.run.smithi135.stdout:[client.mirror.0] 2024-09-14T08:33:00.210 INFO:teuthology.orchestra.run.smithi135.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-14T08:33:00.210 INFO:teuthology.orchestra.run.smithi135.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-14T08:33:00.210 INFO:teuthology.orchestra.run.smithi135.stdout:[client.mirror.1] 2024-09-14T08:33:00.210 INFO:teuthology.orchestra.run.smithi135.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-14T08:33:00.210 INFO:teuthology.orchestra.run.smithi135.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-14T08:33:00.210 INFO:teuthology.orchestra.run.smithi135.stdout:[client.mirror.2] 2024-09-14T08:33:00.210 INFO:teuthology.orchestra.run.smithi135.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-14T08:33:00.210 INFO:teuthology.orchestra.run.smithi135.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-14T08:33:00.211 INFO:teuthology.orchestra.run.smithi135.stdout:[client.mirror.3] 2024-09-14T08:33:00.211 INFO:teuthology.orchestra.run.smithi135.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-14T08:33:00.211 INFO:teuthology.orchestra.run.smithi135.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-14T08:33:00.234 INFO:teuthology.orchestra.run.smithi164.stdout:[global] 2024-09-14T08:33:00.234 INFO:teuthology.orchestra.run.smithi164.stdout: chdir = "" 2024-09-14T08:33:00.234 INFO:teuthology.orchestra.run.smithi164.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2024-09-14T08:33:00.234 INFO:teuthology.orchestra.run.smithi164.stdout: auth supported = cephx 2024-09-14T08:33:00.234 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:33:00.234 INFO:teuthology.orchestra.run.smithi164.stdout: filestore xattr use omap = true 2024-09-14T08:33:00.234 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:33:00.234 INFO:teuthology.orchestra.run.smithi164.stdout: mon clock drift allowed = 1.000 2024-09-14T08:33:00.234 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:33:00.234 INFO:teuthology.orchestra.run.smithi164.stdout: osd crush chooseleaf type = 0 2024-09-14T08:33:00.234 INFO:teuthology.orchestra.run.smithi164.stdout: auth debug = true 2024-09-14T08:33:00.235 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:33:00.235 INFO:teuthology.orchestra.run.smithi164.stdout: ms die on old message = true 2024-09-14T08:33:00.235 INFO:teuthology.orchestra.run.smithi164.stdout: ms die on bug = true 2024-09-14T08:33:00.235 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:33:00.235 INFO:teuthology.orchestra.run.smithi164.stdout: mon max pg per osd = 10000 # >= luminous 2024-09-14T08:33:00.235 INFO:teuthology.orchestra.run.smithi164.stdout: mon pg warn max object skew = 0 2024-09-14T08:33:00.235 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:33:00.235 INFO:teuthology.orchestra.run.smithi164.stdout: # disable pg_autoscaler by default for new pools 2024-09-14T08:33:00.235 INFO:teuthology.orchestra.run.smithi164.stdout: osd_pool_default_pg_autoscale_mode = off 2024-09-14T08:33:00.235 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:33:00.235 INFO:teuthology.orchestra.run.smithi164.stdout: osd pool default size = 2 2024-09-14T08:33:00.235 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:33:00.235 INFO:teuthology.orchestra.run.smithi164.stdout: mon osd allow primary affinity = true 2024-09-14T08:33:00.235 INFO:teuthology.orchestra.run.smithi164.stdout: mon osd allow pg remap = true 2024-09-14T08:33:00.236 INFO:teuthology.orchestra.run.smithi164.stdout: mon warn on legacy crush tunables = false 2024-09-14T08:33:00.236 INFO:teuthology.orchestra.run.smithi164.stdout: mon warn on crush straw calc version zero = false 2024-09-14T08:33:00.236 INFO:teuthology.orchestra.run.smithi164.stdout: mon warn on no sortbitwise = false 2024-09-14T08:33:00.236 INFO:teuthology.orchestra.run.smithi164.stdout: mon warn on osd down out interval zero = false 2024-09-14T08:33:00.236 INFO:teuthology.orchestra.run.smithi164.stdout: mon warn on too few osds = false 2024-09-14T08:33:00.236 INFO:teuthology.orchestra.run.smithi164.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2024-09-14T08:33:00.236 INFO:teuthology.orchestra.run.smithi164.stdout: mon_warn_on_pool_no_redundancy = false 2024-09-14T08:33:00.236 INFO:teuthology.orchestra.run.smithi164.stdout: mon_allow_pool_size_one = true 2024-09-14T08:33:00.236 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:33:00.236 INFO:teuthology.orchestra.run.smithi164.stdout: osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2024-09-14T08:33:00.236 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:33:00.236 INFO:teuthology.orchestra.run.smithi164.stdout: osd default data pool replay window = 5 2024-09-14T08:33:00.236 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:33:00.236 INFO:teuthology.orchestra.run.smithi164.stdout: mon allow pool delete = true 2024-09-14T08:33:00.237 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:33:00.237 INFO:teuthology.orchestra.run.smithi164.stdout: mon cluster log file level = debug 2024-09-14T08:33:00.237 INFO:teuthology.orchestra.run.smithi164.stdout: debug asserts on shutdown = true 2024-09-14T08:33:00.237 INFO:teuthology.orchestra.run.smithi164.stdout: mon health detail to clog = false 2024-09-14T08:33:00.237 INFO:teuthology.orchestra.run.smithi164.stdout: mon host = 172.21.15.164 2024-09-14T08:33:00.237 INFO:teuthology.orchestra.run.smithi164.stdout: mon client directed command retry = 5 2024-09-14T08:33:00.237 INFO:teuthology.orchestra.run.smithi164.stdout: mon warn on pool no app = False 2024-09-14T08:33:00.237 INFO:teuthology.orchestra.run.smithi164.stdout: ms inject socket failures = 5000 2024-09-14T08:33:00.237 INFO:teuthology.orchestra.run.smithi164.stdout: fsid = dd8fb0de-ee38-4627-a696-b631746d355c 2024-09-14T08:33:00.237 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:33:00.237 INFO:teuthology.orchestra.run.smithi164.stdout:[osd] 2024-09-14T08:33:00.237 INFO:teuthology.orchestra.run.smithi164.stdout: osd journal size = 100 2024-09-14T08:33:00.237 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:33:00.237 INFO:teuthology.orchestra.run.smithi164.stdout: osd scrub load threshold = 5.0 2024-09-14T08:33:00.238 INFO:teuthology.orchestra.run.smithi164.stdout: osd scrub max interval = 600 2024-09-14T08:33:00.238 INFO:teuthology.orchestra.run.smithi164.stdout: osd mclock profile = high_recovery_ops 2024-09-14T08:33:00.238 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:33:00.238 INFO:teuthology.orchestra.run.smithi164.stdout: osd recover clone overlap = true 2024-09-14T08:33:00.238 INFO:teuthology.orchestra.run.smithi164.stdout: osd recovery max chunk = 1048576 2024-09-14T08:33:00.238 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:33:00.238 INFO:teuthology.orchestra.run.smithi164.stdout: osd debug shutdown = true 2024-09-14T08:33:00.238 INFO:teuthology.orchestra.run.smithi164.stdout: osd debug op order = true 2024-09-14T08:33:00.238 INFO:teuthology.orchestra.run.smithi164.stdout: osd debug verify stray on activate = true 2024-09-14T08:33:00.238 INFO:teuthology.orchestra.run.smithi164.stdout: osd debug trim objects = true 2024-09-14T08:33:00.238 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:33:00.238 INFO:teuthology.orchestra.run.smithi164.stdout: osd open classes on start = true 2024-09-14T08:33:00.238 INFO:teuthology.orchestra.run.smithi164.stdout: osd debug pg log writeout = true 2024-09-14T08:33:00.238 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:33:00.238 INFO:teuthology.orchestra.run.smithi164.stdout: osd deep scrub update digest min age = 30 2024-09-14T08:33:00.238 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:33:00.239 INFO:teuthology.orchestra.run.smithi164.stdout: osd map max advance = 10 2024-09-14T08:33:00.239 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:33:00.239 INFO:teuthology.orchestra.run.smithi164.stdout: journal zero on create = true 2024-09-14T08:33:00.239 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:33:00.239 INFO:teuthology.orchestra.run.smithi164.stdout: filestore ondisk finisher threads = 3 2024-09-14T08:33:00.239 INFO:teuthology.orchestra.run.smithi164.stdout: filestore apply finisher threads = 3 2024-09-14T08:33:00.239 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:33:00.239 INFO:teuthology.orchestra.run.smithi164.stdout: bdev debug aio = true 2024-09-14T08:33:00.239 INFO:teuthology.orchestra.run.smithi164.stdout: osd debug misdirected ops = true 2024-09-14T08:33:00.239 INFO:teuthology.orchestra.run.smithi164.stdout: bluestore block size = 96636764160 2024-09-14T08:33:00.239 INFO:teuthology.orchestra.run.smithi164.stdout: bluestore compression algorithm = zlib 2024-09-14T08:33:00.239 INFO:teuthology.orchestra.run.smithi164.stdout: bluestore compression mode = aggressive 2024-09-14T08:33:00.239 INFO:teuthology.orchestra.run.smithi164.stdout: bluestore fsck on mount = True 2024-09-14T08:33:00.239 INFO:teuthology.orchestra.run.smithi164.stdout: debug bluefs = 1/20 2024-09-14T08:33:00.240 INFO:teuthology.orchestra.run.smithi164.stdout: debug bluestore = 1/20 2024-09-14T08:33:00.240 INFO:teuthology.orchestra.run.smithi164.stdout: debug ms = 1 2024-09-14T08:33:00.240 INFO:teuthology.orchestra.run.smithi164.stdout: debug osd = 20 2024-09-14T08:33:00.240 INFO:teuthology.orchestra.run.smithi164.stdout: debug rocksdb = 4/10 2024-09-14T08:33:00.240 INFO:teuthology.orchestra.run.smithi164.stdout: mon osd backfillfull_ratio = 0.85 2024-09-14T08:33:00.240 INFO:teuthology.orchestra.run.smithi164.stdout: mon osd full ratio = 0.9 2024-09-14T08:33:00.240 INFO:teuthology.orchestra.run.smithi164.stdout: mon osd nearfull ratio = 0.8 2024-09-14T08:33:00.240 INFO:teuthology.orchestra.run.smithi164.stdout: osd failsafe full ratio = 0.95 2024-09-14T08:33:00.240 INFO:teuthology.orchestra.run.smithi164.stdout: osd objectstore = bluestore 2024-09-14T08:33:00.240 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:33:00.240 INFO:teuthology.orchestra.run.smithi164.stdout:[mgr] 2024-09-14T08:33:00.240 INFO:teuthology.orchestra.run.smithi164.stdout: debug ms = 1 2024-09-14T08:33:00.240 INFO:teuthology.orchestra.run.smithi164.stdout: debug mgr = 20 2024-09-14T08:33:00.240 INFO:teuthology.orchestra.run.smithi164.stdout: debug mon = 20 2024-09-14T08:33:00.241 INFO:teuthology.orchestra.run.smithi164.stdout: debug auth = 20 2024-09-14T08:33:00.241 INFO:teuthology.orchestra.run.smithi164.stdout: mon reweight min pgs per osd = 4 2024-09-14T08:33:00.241 INFO:teuthology.orchestra.run.smithi164.stdout: mon reweight min bytes per osd = 10 2024-09-14T08:33:00.241 INFO:teuthology.orchestra.run.smithi164.stdout: mgr/telemetry/nag = false 2024-09-14T08:33:00.241 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:33:00.241 INFO:teuthology.orchestra.run.smithi164.stdout:[mon] 2024-09-14T08:33:00.241 INFO:teuthology.orchestra.run.smithi164.stdout: debug ms = 1 2024-09-14T08:33:00.241 INFO:teuthology.orchestra.run.smithi164.stdout: debug mon = 20 2024-09-14T08:33:00.241 INFO:teuthology.orchestra.run.smithi164.stdout: debug paxos = 20 2024-09-14T08:33:00.241 INFO:teuthology.orchestra.run.smithi164.stdout: debug auth = 20 2024-09-14T08:33:00.241 INFO:teuthology.orchestra.run.smithi164.stdout: mon data avail warn = 5 2024-09-14T08:33:00.241 INFO:teuthology.orchestra.run.smithi164.stdout: mon mgr mkfs grace = 240 2024-09-14T08:33:00.241 INFO:teuthology.orchestra.run.smithi164.stdout: mon reweight min pgs per osd = 4 2024-09-14T08:33:00.241 INFO:teuthology.orchestra.run.smithi164.stdout: mon osd reporter subtree level = osd 2024-09-14T08:33:00.242 INFO:teuthology.orchestra.run.smithi164.stdout: mon osd prime pg temp = true 2024-09-14T08:33:00.242 INFO:teuthology.orchestra.run.smithi164.stdout: mon reweight min bytes per osd = 10 2024-09-14T08:33:00.242 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:33:00.242 INFO:teuthology.orchestra.run.smithi164.stdout: # rotate auth tickets quickly to exercise renewal paths 2024-09-14T08:33:00.242 INFO:teuthology.orchestra.run.smithi164.stdout: auth mon ticket ttl = 660 # 11m 2024-09-14T08:33:00.242 INFO:teuthology.orchestra.run.smithi164.stdout: auth service ticket ttl = 240 # 4m 2024-09-14T08:33:00.242 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:33:00.242 INFO:teuthology.orchestra.run.smithi164.stdout: # don't complain about insecure global_id in the test suite 2024-09-14T08:33:00.242 INFO:teuthology.orchestra.run.smithi164.stdout: mon_warn_on_insecure_global_id_reclaim = false 2024-09-14T08:33:00.242 INFO:teuthology.orchestra.run.smithi164.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2024-09-14T08:33:00.242 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:33:00.242 INFO:teuthology.orchestra.run.smithi164.stdout: # 1m isn't quite enough 2024-09-14T08:33:00.242 INFO:teuthology.orchestra.run.smithi164.stdout: mon_down_mkfs_grace = 2m 2024-09-14T08:33:00.242 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:33:00.242 INFO:teuthology.orchestra.run.smithi164.stdout: mon_warn_on_filestore_osds = false 2024-09-14T08:33:00.243 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:33:00.243 INFO:teuthology.orchestra.run.smithi164.stdout:[client] 2024-09-14T08:33:00.243 INFO:teuthology.orchestra.run.smithi164.stdout: rgw cache enabled = true 2024-09-14T08:33:00.243 INFO:teuthology.orchestra.run.smithi164.stdout: rgw enable ops log = true 2024-09-14T08:33:00.243 INFO:teuthology.orchestra.run.smithi164.stdout: rgw enable usage log = true 2024-09-14T08:33:00.243 INFO:teuthology.orchestra.run.smithi164.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2024-09-14T08:33:00.243 INFO:teuthology.orchestra.run.smithi164.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2024-09-14T08:33:00.243 INFO:teuthology.orchestra.run.smithi164.stdout: debug rbd = 20 2024-09-14T08:33:00.243 INFO:teuthology.orchestra.run.smithi164.stdout: debug rbd_mirror = 15 2024-09-14T08:33:00.243 INFO:teuthology.orchestra.run.smithi164.stdout: log to stderr = False 2024-09-14T08:33:00.243 INFO:teuthology.orchestra.run.smithi164.stdout: rbd default features = 125 2024-09-14T08:33:00.243 INFO:teuthology.orchestra.run.smithi164.stdout: rbd mirror image policy type = simple 2024-09-14T08:33:00.243 INFO:teuthology.orchestra.run.smithi164.stdout:[mon.a] 2024-09-14T08:33:00.243 INFO:teuthology.orchestra.run.smithi164.stdout:[client.mirror.0] 2024-09-14T08:33:00.243 INFO:teuthology.orchestra.run.smithi164.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-14T08:33:00.243 INFO:teuthology.orchestra.run.smithi164.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-14T08:33:00.244 INFO:teuthology.orchestra.run.smithi164.stdout:[client.mirror.1] 2024-09-14T08:33:00.244 INFO:teuthology.orchestra.run.smithi164.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-14T08:33:00.244 INFO:teuthology.orchestra.run.smithi164.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-14T08:33:00.244 INFO:teuthology.orchestra.run.smithi164.stdout:[client.mirror.2] 2024-09-14T08:33:00.244 INFO:teuthology.orchestra.run.smithi164.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-14T08:33:00.244 INFO:teuthology.orchestra.run.smithi164.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-14T08:33:00.244 INFO:teuthology.orchestra.run.smithi164.stdout:[client.mirror.3] 2024-09-14T08:33:00.244 INFO:teuthology.orchestra.run.smithi164.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-14T08:33:00.244 INFO:teuthology.orchestra.run.smithi164.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-14T08:33:00.245 INFO:tasks.ceph:Creating admin key on cluster2.mon.a... 2024-09-14T08:33:00.246 DEBUG:teuthology.orchestra.run.smithi164:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *' /etc/ceph/cluster2.keyring 2024-09-14T08:33:00.340 INFO:tasks.ceph:Copying monmap to all nodes... 2024-09-14T08:33:00.340 DEBUG:teuthology.orchestra.run.smithi164:> set -ex 2024-09-14T08:33:00.341 DEBUG:teuthology.orchestra.run.smithi164:> dd if=/etc/ceph/cluster2.keyring of=/dev/stdout 2024-09-14T08:33:00.356 DEBUG:teuthology.orchestra.run.smithi164:> set -ex 2024-09-14T08:33:00.356 DEBUG:teuthology.orchestra.run.smithi164:> dd if=/home/ubuntu/cephtest/cluster2.monmap of=/dev/stdout 2024-09-14T08:33:00.410 INFO:tasks.ceph:Sending monmap to node ubuntu@smithi135.front.sepia.ceph.com 2024-09-14T08:33:00.411 DEBUG:teuthology.orchestra.run.smithi135:> set -ex 2024-09-14T08:33:00.411 DEBUG:teuthology.orchestra.run.smithi135:> sudo dd of=/etc/ceph/cluster2.keyring 2024-09-14T08:33:00.411 DEBUG:teuthology.orchestra.run.smithi135:> sudo chmod 0644 /etc/ceph/cluster2.keyring 2024-09-14T08:33:00.448 DEBUG:teuthology.orchestra.run.smithi135:> set -ex 2024-09-14T08:33:00.448 DEBUG:teuthology.orchestra.run.smithi135:> dd of=/home/ubuntu/cephtest/cluster2.monmap 2024-09-14T08:33:00.504 INFO:tasks.ceph:Sending monmap to node ubuntu@smithi164.front.sepia.ceph.com 2024-09-14T08:33:00.504 DEBUG:teuthology.orchestra.run.smithi164:> set -ex 2024-09-14T08:33:00.505 DEBUG:teuthology.orchestra.run.smithi164:> sudo dd of=/etc/ceph/cluster2.keyring 2024-09-14T08:33:00.505 DEBUG:teuthology.orchestra.run.smithi164:> sudo chmod 0644 /etc/ceph/cluster2.keyring 2024-09-14T08:33:00.539 DEBUG:teuthology.orchestra.run.smithi164:> set -ex 2024-09-14T08:33:00.539 DEBUG:teuthology.orchestra.run.smithi164:> dd of=/home/ubuntu/cephtest/cluster2.monmap 2024-09-14T08:33:00.595 INFO:tasks.ceph:Setting up mon nodes... 2024-09-14T08:33:00.595 INFO:tasks.ceph:Setting up mgr nodes... 2024-09-14T08:33:00.595 DEBUG:teuthology.orchestra.run.smithi135:> sudo mkdir -p /var/lib/ceph/mgr/cluster2-x && sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=mgr.x /var/lib/ceph/mgr/cluster2-x/keyring 2024-09-14T08:33:00.647 INFO:teuthology.orchestra.run.smithi135.stdout:creating /var/lib/ceph/mgr/cluster2-x/keyring 2024-09-14T08:33:00.649 INFO:tasks.ceph:Setting up mds nodes... 2024-09-14T08:33:00.650 INFO:tasks.ceph_client:Setting up client nodes... 2024-09-14T08:33:00.650 DEBUG:teuthology.orchestra.run.smithi135:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.0 /etc/ceph/cluster2.client.0.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.0.keyring 2024-09-14T08:33:00.691 INFO:teuthology.orchestra.run.smithi135.stdout:creating /etc/ceph/cluster2.client.0.keyring 2024-09-14T08:33:00.704 DEBUG:teuthology.orchestra.run.smithi164:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror /etc/ceph/cluster2.client.mirror.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.keyring 2024-09-14T08:33:00.745 INFO:teuthology.orchestra.run.smithi164.stdout:creating /etc/ceph/cluster2.client.mirror.keyring 2024-09-14T08:33:00.757 DEBUG:teuthology.orchestra.run.smithi164:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.0 /etc/ceph/cluster2.client.mirror.0.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.0.keyring 2024-09-14T08:33:00.838 INFO:teuthology.orchestra.run.smithi164.stdout:creating /etc/ceph/cluster2.client.mirror.0.keyring 2024-09-14T08:33:00.851 DEBUG:teuthology.orchestra.run.smithi164:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.1 /etc/ceph/cluster2.client.mirror.1.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.1.keyring 2024-09-14T08:33:00.932 INFO:teuthology.orchestra.run.smithi164.stdout:creating /etc/ceph/cluster2.client.mirror.1.keyring 2024-09-14T08:33:00.945 DEBUG:teuthology.orchestra.run.smithi164:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.2 /etc/ceph/cluster2.client.mirror.2.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.2.keyring 2024-09-14T08:33:01.027 INFO:teuthology.orchestra.run.smithi164.stdout:creating /etc/ceph/cluster2.client.mirror.2.keyring 2024-09-14T08:33:01.040 DEBUG:teuthology.orchestra.run.smithi164:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.3 /etc/ceph/cluster2.client.mirror.3.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.3.keyring 2024-09-14T08:33:01.121 INFO:teuthology.orchestra.run.smithi164.stdout:creating /etc/ceph/cluster2.client.mirror.3.keyring 2024-09-14T08:33:01.133 DEBUG:teuthology.orchestra.run.smithi164:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.4 /etc/ceph/cluster2.client.mirror.4.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.4.keyring 2024-09-14T08:33:01.215 INFO:teuthology.orchestra.run.smithi164.stdout:creating /etc/ceph/cluster2.client.mirror.4.keyring 2024-09-14T08:33:01.228 DEBUG:teuthology.orchestra.run.smithi164:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.5 /etc/ceph/cluster2.client.mirror.5.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.5.keyring 2024-09-14T08:33:01.308 INFO:teuthology.orchestra.run.smithi164.stdout:creating /etc/ceph/cluster2.client.mirror.5.keyring 2024-09-14T08:33:01.321 DEBUG:teuthology.orchestra.run.smithi164:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.6 /etc/ceph/cluster2.client.mirror.6.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.6.keyring 2024-09-14T08:33:01.402 INFO:teuthology.orchestra.run.smithi164.stdout:creating /etc/ceph/cluster2.client.mirror.6.keyring 2024-09-14T08:33:01.414 INFO:tasks.ceph:Running mkfs on osd nodes... 2024-09-14T08:33:01.414 INFO:tasks.ceph:ctx.disk_config.remote_to_roles_to_dev: {Remote(name='ubuntu@smithi135.front.sepia.ceph.com'): {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'}, Remote(name='ubuntu@smithi164.front.sepia.ceph.com'): {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'}} 2024-09-14T08:33:01.415 DEBUG:teuthology.orchestra.run.smithi164:> sudo mkdir -p /var/lib/ceph/osd/cluster2-0 2024-09-14T08:33:01.479 INFO:tasks.ceph:roles_to_devs: {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'} 2024-09-14T08:33:01.479 INFO:tasks.ceph:role: cluster2.osd.0 2024-09-14T08:33:01.479 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_1 on ubuntu@smithi164.front.sepia.ceph.com 2024-09-14T08:33:01.480 DEBUG:teuthology.orchestra.run.smithi164:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_1 2024-09-14T08:33:01.551 INFO:teuthology.orchestra.run.smithi164.stdout:meta-data=/dev/vg_nvme/lv_1 isize=2048 agcount=4, agsize=5859072 blks 2024-09-14T08:33:01.552 INFO:teuthology.orchestra.run.smithi164.stdout: = sectsz=512 attr=2, projid32bit=1 2024-09-14T08:33:01.552 INFO:teuthology.orchestra.run.smithi164.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2024-09-14T08:33:01.552 INFO:teuthology.orchestra.run.smithi164.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2024-09-14T08:33:01.552 INFO:teuthology.orchestra.run.smithi164.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2024-09-14T08:33:01.552 INFO:teuthology.orchestra.run.smithi164.stdout: = sunit=0 swidth=0 blks 2024-09-14T08:33:01.552 INFO:teuthology.orchestra.run.smithi164.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2024-09-14T08:33:01.552 INFO:teuthology.orchestra.run.smithi164.stdout:log =internal log bsize=4096 blocks=16384, version=2 2024-09-14T08:33:01.552 INFO:teuthology.orchestra.run.smithi164.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2024-09-14T08:33:01.552 INFO:teuthology.orchestra.run.smithi164.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2024-09-14T08:33:02.058 INFO:teuthology.orchestra.run.smithi164.stdout:Discarding blocks...Done. 2024-09-14T08:33:02.060 INFO:tasks.ceph:mount /dev/vg_nvme/lv_1 on ubuntu@smithi164.front.sepia.ceph.com -o noatime 2024-09-14T08:33:02.061 DEBUG:teuthology.orchestra.run.smithi164:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_1 /var/lib/ceph/osd/cluster2-0 2024-09-14T08:33:02.202 DEBUG:teuthology.orchestra.run.smithi164:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster2-0 2024-09-14T08:33:02.234 DEBUG:teuthology.orchestra.run.smithi164:> sudo mkdir -p /var/lib/ceph/osd/cluster2-1 2024-09-14T08:33:02.300 INFO:tasks.ceph:roles_to_devs: {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'} 2024-09-14T08:33:02.300 INFO:tasks.ceph:role: cluster2.osd.1 2024-09-14T08:33:02.300 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_2 on ubuntu@smithi164.front.sepia.ceph.com 2024-09-14T08:33:02.300 DEBUG:teuthology.orchestra.run.smithi164:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_2 2024-09-14T08:33:02.373 INFO:teuthology.orchestra.run.smithi164.stdout:meta-data=/dev/vg_nvme/lv_2 isize=2048 agcount=4, agsize=5859072 blks 2024-09-14T08:33:02.373 INFO:teuthology.orchestra.run.smithi164.stdout: = sectsz=512 attr=2, projid32bit=1 2024-09-14T08:33:02.373 INFO:teuthology.orchestra.run.smithi164.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2024-09-14T08:33:02.373 INFO:teuthology.orchestra.run.smithi164.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2024-09-14T08:33:02.373 INFO:teuthology.orchestra.run.smithi164.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2024-09-14T08:33:02.373 INFO:teuthology.orchestra.run.smithi164.stdout: = sunit=0 swidth=0 blks 2024-09-14T08:33:02.373 INFO:teuthology.orchestra.run.smithi164.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2024-09-14T08:33:02.373 INFO:teuthology.orchestra.run.smithi164.stdout:log =internal log bsize=4096 blocks=16384, version=2 2024-09-14T08:33:02.373 INFO:teuthology.orchestra.run.smithi164.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2024-09-14T08:33:02.374 INFO:teuthology.orchestra.run.smithi164.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2024-09-14T08:33:02.829 INFO:teuthology.orchestra.run.smithi164.stdout:Discarding blocks...Done. 2024-09-14T08:33:02.831 INFO:tasks.ceph:mount /dev/vg_nvme/lv_2 on ubuntu@smithi164.front.sepia.ceph.com -o noatime 2024-09-14T08:33:02.831 DEBUG:teuthology.orchestra.run.smithi164:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_2 /var/lib/ceph/osd/cluster2-1 2024-09-14T08:33:02.924 DEBUG:teuthology.orchestra.run.smithi164:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster2-1 2024-09-14T08:33:02.952 DEBUG:teuthology.orchestra.run.smithi164:> sudo mkdir -p /var/lib/ceph/osd/cluster2-2 2024-09-14T08:33:03.017 INFO:tasks.ceph:roles_to_devs: {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'} 2024-09-14T08:33:03.017 INFO:tasks.ceph:role: cluster2.osd.2 2024-09-14T08:33:03.017 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_3 on ubuntu@smithi164.front.sepia.ceph.com 2024-09-14T08:33:03.017 DEBUG:teuthology.orchestra.run.smithi164:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_3 2024-09-14T08:33:03.089 INFO:teuthology.orchestra.run.smithi164.stdout:meta-data=/dev/vg_nvme/lv_3 isize=2048 agcount=4, agsize=5859072 blks 2024-09-14T08:33:03.089 INFO:teuthology.orchestra.run.smithi164.stdout: = sectsz=512 attr=2, projid32bit=1 2024-09-14T08:33:03.089 INFO:teuthology.orchestra.run.smithi164.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2024-09-14T08:33:03.089 INFO:teuthology.orchestra.run.smithi164.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2024-09-14T08:33:03.089 INFO:teuthology.orchestra.run.smithi164.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2024-09-14T08:33:03.090 INFO:teuthology.orchestra.run.smithi164.stdout: = sunit=0 swidth=0 blks 2024-09-14T08:33:03.090 INFO:teuthology.orchestra.run.smithi164.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2024-09-14T08:33:03.090 INFO:teuthology.orchestra.run.smithi164.stdout:log =internal log bsize=4096 blocks=16384, version=2 2024-09-14T08:33:03.090 INFO:teuthology.orchestra.run.smithi164.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2024-09-14T08:33:03.090 INFO:teuthology.orchestra.run.smithi164.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2024-09-14T08:33:03.577 INFO:teuthology.orchestra.run.smithi164.stdout:Discarding blocks...Done. 2024-09-14T08:33:03.579 INFO:tasks.ceph:mount /dev/vg_nvme/lv_3 on ubuntu@smithi164.front.sepia.ceph.com -o noatime 2024-09-14T08:33:03.579 DEBUG:teuthology.orchestra.run.smithi164:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_3 /var/lib/ceph/osd/cluster2-2 2024-09-14T08:33:03.723 DEBUG:teuthology.orchestra.run.smithi164:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster2-2 2024-09-14T08:33:03.753 DEBUG:teuthology.orchestra.run.smithi164:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster2 --mkfs --mkkey -i 0 --monmap /home/ubuntu/cephtest/cluster2.monmap 2024-09-14T08:33:03.845 INFO:teuthology.orchestra.run.smithi164.stderr:2024-09-14T08:33:03.845+0000 7f57b722c640 -1 auth: error reading file: /var/lib/ceph/osd/cluster2-0/keyring: can't open /var/lib/ceph/osd/cluster2-0/keyring: (2) No such file or directory 2024-09-14T08:33:03.851 INFO:teuthology.orchestra.run.smithi164.stderr:2024-09-14T08:33:03.850+0000 7f57b722c640 -1 created new key in keyring /var/lib/ceph/osd/cluster2-0/keyring 2024-09-14T08:33:03.851 INFO:teuthology.orchestra.run.smithi164.stderr:2024-09-14T08:33:03.850+0000 7f57b722c640 -1 bdev(0x55e6b5cbf800 /var/lib/ceph/osd/cluster2-0/block) open stat got: (1) Operation not permitted 2024-09-14T08:33:03.851 INFO:teuthology.orchestra.run.smithi164.stderr:2024-09-14T08:33:03.850+0000 7f57b722c640 -1 bluestore(/var/lib/ceph/osd/cluster2-0) _read_fsid unparsable uuid 2024-09-14T08:33:06.755 DEBUG:teuthology.orchestra.run.smithi164:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster2-0 2024-09-14T08:33:06.783 DEBUG:teuthology.orchestra.run.smithi164:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster2 --mkfs --mkkey -i 1 --monmap /home/ubuntu/cephtest/cluster2.monmap 2024-09-14T08:33:06.870 INFO:teuthology.orchestra.run.smithi164.stderr:2024-09-14T08:33:06.869+0000 7f2e12418640 -1 auth: error reading file: /var/lib/ceph/osd/cluster2-1/keyring: can't open /var/lib/ceph/osd/cluster2-1/keyring: (2) No such file or directory 2024-09-14T08:33:06.871 INFO:teuthology.orchestra.run.smithi164.stderr:2024-09-14T08:33:06.870+0000 7f2e12418640 -1 created new key in keyring /var/lib/ceph/osd/cluster2-1/keyring 2024-09-14T08:33:06.871 INFO:teuthology.orchestra.run.smithi164.stderr:2024-09-14T08:33:06.870+0000 7f2e12418640 -1 bdev(0x56517aab9800 /var/lib/ceph/osd/cluster2-1/block) open stat got: (1) Operation not permitted 2024-09-14T08:33:06.871 INFO:teuthology.orchestra.run.smithi164.stderr:2024-09-14T08:33:06.870+0000 7f2e12418640 -1 bluestore(/var/lib/ceph/osd/cluster2-1) _read_fsid unparsable uuid 2024-09-14T08:33:10.037 DEBUG:teuthology.orchestra.run.smithi164:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster2-1 2024-09-14T08:33:10.063 DEBUG:teuthology.orchestra.run.smithi164:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster2 --mkfs --mkkey -i 2 --monmap /home/ubuntu/cephtest/cluster2.monmap 2024-09-14T08:33:10.150 INFO:teuthology.orchestra.run.smithi164.stderr:2024-09-14T08:33:10.149+0000 7fcad020f640 -1 auth: error reading file: /var/lib/ceph/osd/cluster2-2/keyring: can't open /var/lib/ceph/osd/cluster2-2/keyring: (2) No such file or directory 2024-09-14T08:33:10.150 INFO:teuthology.orchestra.run.smithi164.stderr:2024-09-14T08:33:10.150+0000 7fcad020f640 -1 created new key in keyring /var/lib/ceph/osd/cluster2-2/keyring 2024-09-14T08:33:10.150 INFO:teuthology.orchestra.run.smithi164.stderr:2024-09-14T08:33:10.150+0000 7fcad020f640 -1 bdev(0x5556b0b69800 /var/lib/ceph/osd/cluster2-2/block) open stat got: (1) Operation not permitted 2024-09-14T08:33:10.151 INFO:teuthology.orchestra.run.smithi164.stderr:2024-09-14T08:33:10.150+0000 7fcad020f640 -1 bluestore(/var/lib/ceph/osd/cluster2-2) _read_fsid unparsable uuid 2024-09-14T08:33:13.559 DEBUG:teuthology.orchestra.run.smithi164:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster2-2 2024-09-14T08:33:13.585 INFO:tasks.ceph:Reading keys from all nodes... 2024-09-14T08:33:13.585 DEBUG:teuthology.orchestra.run.smithi135:> set -ex 2024-09-14T08:33:13.585 DEBUG:teuthology.orchestra.run.smithi135:> sudo dd if=/var/lib/ceph/mgr/cluster2-x/keyring of=/dev/stdout 2024-09-14T08:33:13.615 DEBUG:teuthology.orchestra.run.smithi164:> set -ex 2024-09-14T08:33:13.615 DEBUG:teuthology.orchestra.run.smithi164:> sudo dd if=/var/lib/ceph/osd/cluster2-0/keyring of=/dev/stdout 2024-09-14T08:33:13.649 DEBUG:teuthology.orchestra.run.smithi164:> set -ex 2024-09-14T08:33:13.649 DEBUG:teuthology.orchestra.run.smithi164:> sudo dd if=/var/lib/ceph/osd/cluster2-1/keyring of=/dev/stdout 2024-09-14T08:33:13.714 DEBUG:teuthology.orchestra.run.smithi164:> set -ex 2024-09-14T08:33:13.714 DEBUG:teuthology.orchestra.run.smithi164:> sudo dd if=/var/lib/ceph/osd/cluster2-2/keyring of=/dev/stdout 2024-09-14T08:33:13.778 DEBUG:teuthology.orchestra.run.smithi135:> set -ex 2024-09-14T08:33:13.778 DEBUG:teuthology.orchestra.run.smithi135:> dd if=/etc/ceph/cluster2.client.0.keyring of=/dev/stdout 2024-09-14T08:33:13.797 DEBUG:teuthology.orchestra.run.smithi164:> set -ex 2024-09-14T08:33:13.860 DEBUG:teuthology.orchestra.run.smithi164:> dd if=/etc/ceph/cluster2.client.mirror.keyring of=/dev/stdout 2024-09-14T08:33:13.876 DEBUG:teuthology.orchestra.run.smithi164:> set -ex 2024-09-14T08:33:13.876 DEBUG:teuthology.orchestra.run.smithi164:> dd if=/etc/ceph/cluster2.client.mirror.0.keyring of=/dev/stdout 2024-09-14T08:33:13.932 DEBUG:teuthology.orchestra.run.smithi164:> set -ex 2024-09-14T08:33:13.932 DEBUG:teuthology.orchestra.run.smithi164:> dd if=/etc/ceph/cluster2.client.mirror.1.keyring of=/dev/stdout 2024-09-14T08:33:13.987 DEBUG:teuthology.orchestra.run.smithi164:> set -ex 2024-09-14T08:33:13.987 DEBUG:teuthology.orchestra.run.smithi164:> dd if=/etc/ceph/cluster2.client.mirror.2.keyring of=/dev/stdout 2024-09-14T08:33:14.042 DEBUG:teuthology.orchestra.run.smithi164:> set -ex 2024-09-14T08:33:14.042 DEBUG:teuthology.orchestra.run.smithi164:> dd if=/etc/ceph/cluster2.client.mirror.3.keyring of=/dev/stdout 2024-09-14T08:33:14.098 DEBUG:teuthology.orchestra.run.smithi164:> set -ex 2024-09-14T08:33:14.098 DEBUG:teuthology.orchestra.run.smithi164:> dd if=/etc/ceph/cluster2.client.mirror.4.keyring of=/dev/stdout 2024-09-14T08:33:14.152 DEBUG:teuthology.orchestra.run.smithi164:> set -ex 2024-09-14T08:33:14.152 DEBUG:teuthology.orchestra.run.smithi164:> dd if=/etc/ceph/cluster2.client.mirror.5.keyring of=/dev/stdout 2024-09-14T08:33:14.208 DEBUG:teuthology.orchestra.run.smithi164:> set -ex 2024-09-14T08:33:14.208 DEBUG:teuthology.orchestra.run.smithi164:> dd if=/etc/ceph/cluster2.client.mirror.6.keyring of=/dev/stdout 2024-09-14T08:33:14.263 INFO:tasks.ceph:Adding keys to all mons... 2024-09-14T08:33:14.263 DEBUG:teuthology.orchestra.run.smithi164:> sudo tee -a /etc/ceph/cluster2.keyring 2024-09-14T08:33:14.326 INFO:teuthology.orchestra.run.smithi164.stdout:[mgr.x] 2024-09-14T08:33:14.327 INFO:teuthology.orchestra.run.smithi164.stdout: key = AQA8SuVm+fqpJhAAiI4D0oLed2hnpGTKAWs+/A== 2024-09-14T08:33:14.327 INFO:teuthology.orchestra.run.smithi164.stdout:[osd.0] 2024-09-14T08:33:14.327 INFO:teuthology.orchestra.run.smithi164.stdout: key = AQA/SuVmSsx5MhAAm09+Ivhpb8N+Alq712qprQ== 2024-09-14T08:33:14.327 INFO:teuthology.orchestra.run.smithi164.stdout:[osd.1] 2024-09-14T08:33:14.327 INFO:teuthology.orchestra.run.smithi164.stdout: key = AQBCSuVmJ0r1MxAAj7oHGAPpk66xcahAw1oicg== 2024-09-14T08:33:14.327 INFO:teuthology.orchestra.run.smithi164.stdout:[osd.2] 2024-09-14T08:33:14.327 INFO:teuthology.orchestra.run.smithi164.stdout: key = AQBGSuVm3rYICRAAhHOD0ra7CwLjTgRFw/+wlg== 2024-09-14T08:33:14.327 INFO:teuthology.orchestra.run.smithi164.stdout:[client.0] 2024-09-14T08:33:14.327 INFO:teuthology.orchestra.run.smithi164.stdout: key = AQA8SuVmRJVHKRAA2kv9qBRCYiBzNcllva2QBA== 2024-09-14T08:33:14.327 INFO:teuthology.orchestra.run.smithi164.stdout:[client.mirror] 2024-09-14T08:33:14.327 INFO:teuthology.orchestra.run.smithi164.stdout: key = AQA8SuVmyDeDLBAAtB27M66Y1S7CzX1iEl8gLw== 2024-09-14T08:33:14.327 INFO:teuthology.orchestra.run.smithi164.stdout:[client.mirror.0] 2024-09-14T08:33:14.328 INFO:teuthology.orchestra.run.smithi164.stdout: key = AQA8SuVmZ9EWMhAAXByDQRr+Wu+algMYlyxRVw== 2024-09-14T08:33:14.328 INFO:teuthology.orchestra.run.smithi164.stdout:[client.mirror.1] 2024-09-14T08:33:14.328 INFO:teuthology.orchestra.run.smithi164.stdout: key = AQA8SuVmL1upNxAAALAGO2uYGL1KLA/rcvDDFA== 2024-09-14T08:33:14.328 INFO:teuthology.orchestra.run.smithi164.stdout:[client.mirror.2] 2024-09-14T08:33:14.328 INFO:teuthology.orchestra.run.smithi164.stdout: key = AQA9SuVmU4yyARAA4bQupWUKxLRYUiw90CMmbg== 2024-09-14T08:33:14.328 INFO:teuthology.orchestra.run.smithi164.stdout:[client.mirror.3] 2024-09-14T08:33:14.328 INFO:teuthology.orchestra.run.smithi164.stdout: key = AQA9SuVmyZJMBxAAyN4nTCyrPgIYckUUYnpeog== 2024-09-14T08:33:14.328 INFO:teuthology.orchestra.run.smithi164.stdout:[client.mirror.4] 2024-09-14T08:33:14.328 INFO:teuthology.orchestra.run.smithi164.stdout: key = AQA9SuVmxzDuDBAA8xs3I2j/S2MiG1FPlzircQ== 2024-09-14T08:33:14.328 INFO:teuthology.orchestra.run.smithi164.stdout:[client.mirror.5] 2024-09-14T08:33:14.328 INFO:teuthology.orchestra.run.smithi164.stdout: key = AQA9SuVmvqZ4EhAAY0X9aeABCm6RDfGpG5OjUw== 2024-09-14T08:33:14.328 INFO:teuthology.orchestra.run.smithi164.stdout:[client.mirror.6] 2024-09-14T08:33:14.328 INFO:teuthology.orchestra.run.smithi164.stdout: key = AQA9SuVmiiEJGBAADTPYxljbyLtPRYb+hJiIrQ== 2024-09-14T08:33:14.329 DEBUG:teuthology.orchestra.run.smithi164:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=mgr.x --cap mon 'allow profile mgr' --cap osd 'allow *' --cap mds 'allow *' 2024-09-14T08:33:14.421 DEBUG:teuthology.orchestra.run.smithi164:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=osd.0 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2024-09-14T08:33:14.472 DEBUG:teuthology.orchestra.run.smithi164:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=osd.1 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2024-09-14T08:33:14.523 DEBUG:teuthology.orchestra.run.smithi164:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=osd.2 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2024-09-14T08:33:14.574 DEBUG:teuthology.orchestra.run.smithi164:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-09-14T08:33:14.628 DEBUG:teuthology.orchestra.run.smithi164:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-09-14T08:33:14.681 DEBUG:teuthology.orchestra.run.smithi164:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-09-14T08:33:14.735 DEBUG:teuthology.orchestra.run.smithi164:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.1 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-09-14T08:33:14.789 DEBUG:teuthology.orchestra.run.smithi164:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.2 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-09-14T08:33:14.843 DEBUG:teuthology.orchestra.run.smithi164:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.3 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-09-14T08:33:14.896 DEBUG:teuthology.orchestra.run.smithi164:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.4 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-09-14T08:33:14.950 DEBUG:teuthology.orchestra.run.smithi164:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.5 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-09-14T08:33:15.004 DEBUG:teuthology.orchestra.run.smithi164:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.6 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-09-14T08:33:15.058 INFO:tasks.ceph:Running mkfs on mon nodes... 2024-09-14T08:33:15.058 DEBUG:teuthology.orchestra.run.smithi164:> sudo mkdir -p /var/lib/ceph/mon/cluster2-a 2024-09-14T08:33:15.083 DEBUG:teuthology.orchestra.run.smithi164:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-mon --cluster cluster2 --mkfs -i a --monmap /home/ubuntu/cephtest/cluster2.monmap --keyring /etc/ceph/cluster2.keyring 2024-09-14T08:33:15.179 DEBUG:teuthology.orchestra.run.smithi164:> sudo chown -R ceph:ceph /var/lib/ceph/mon/cluster2-a 2024-09-14T08:33:15.203 DEBUG:teuthology.orchestra.run.smithi164:> rm -- /home/ubuntu/cephtest/cluster2.monmap 2024-09-14T08:33:15.258 INFO:tasks.ceph:Starting mon daemons in cluster cluster2... 2024-09-14T08:33:15.258 INFO:tasks.ceph.cluster2.mon.a:Restarting daemon 2024-09-14T08:33:15.259 DEBUG:teuthology.orchestra.run.smithi164:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f --cluster cluster2 -i a 2024-09-14T08:33:15.300 INFO:tasks.ceph.cluster2.mon.a:Started 2024-09-14T08:33:15.301 INFO:tasks.ceph:Starting mgr daemons in cluster cluster2... 2024-09-14T08:33:15.301 INFO:tasks.ceph.cluster2.mgr.x:Restarting daemon 2024-09-14T08:33:15.301 DEBUG:teuthology.orchestra.run.smithi135:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mgr -f --cluster cluster2 -i x 2024-09-14T08:33:15.304 INFO:tasks.ceph.cluster2.mgr.x:Started 2024-09-14T08:33:15.304 DEBUG:tasks.ceph:set 0 configs 2024-09-14T08:33:15.304 INFO:tasks.ceph:Setting crush tunables to default 2024-09-14T08:33:15.304 DEBUG:teuthology.orchestra.run.smithi164:> sudo ceph --cluster cluster2 osd crush tunables default 2024-09-14T08:33:15.475 INFO:teuthology.orchestra.run.smithi164.stderr:adjusted tunables profile to default 2024-09-14T08:33:15.486 INFO:tasks.ceph:check_enable_crimson: False 2024-09-14T08:33:15.487 INFO:tasks.ceph:Starting osd daemons in cluster cluster2... 2024-09-14T08:33:15.487 DEBUG:teuthology.orchestra.run.smithi164:> set -ex 2024-09-14T08:33:15.487 DEBUG:teuthology.orchestra.run.smithi164:> sudo dd if=/var/lib/ceph/osd/cluster2-0/fsid of=/dev/stdout 2024-09-14T08:33:15.511 DEBUG:teuthology.orchestra.run.smithi164:> set -ex 2024-09-14T08:33:15.512 DEBUG:teuthology.orchestra.run.smithi164:> sudo dd if=/var/lib/ceph/osd/cluster2-1/fsid of=/dev/stdout 2024-09-14T08:33:15.576 DEBUG:teuthology.orchestra.run.smithi164:> set -ex 2024-09-14T08:33:15.576 DEBUG:teuthology.orchestra.run.smithi164:> sudo dd if=/var/lib/ceph/osd/cluster2-2/fsid of=/dev/stdout 2024-09-14T08:33:15.640 DEBUG:teuthology.orchestra.run.smithi164:> sudo ceph --cluster cluster2 osd new 93965f62-149b-4a3a-a65f-78058a6342df 0 2024-09-14T08:33:15.853 INFO:teuthology.orchestra.run.smithi164.stdout:0 2024-09-14T08:33:15.865 DEBUG:teuthology.orchestra.run.smithi164:> sudo ceph --cluster cluster2 osd new dec8040a-7518-4bb9-84b5-0340c673c6bf 1 2024-09-14T08:33:16.039 INFO:teuthology.orchestra.run.smithi164.stdout:1 2024-09-14T08:33:16.050 DEBUG:teuthology.orchestra.run.smithi164:> sudo ceph --cluster cluster2 osd new beca2c1e-834d-4485-b774-ece1dcd06a8d 2 2024-09-14T08:33:16.223 INFO:teuthology.orchestra.run.smithi164.stdout:2 2024-09-14T08:33:16.234 INFO:tasks.ceph.cluster2.osd.0:Restarting daemon 2024-09-14T08:33:16.234 DEBUG:teuthology.orchestra.run.smithi164:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster2 -i 0 2024-09-14T08:33:16.236 INFO:tasks.ceph.cluster2.osd.0:Started 2024-09-14T08:33:16.237 INFO:tasks.ceph.cluster2.osd.1:Restarting daemon 2024-09-14T08:33:16.237 DEBUG:teuthology.orchestra.run.smithi164:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster2 -i 1 2024-09-14T08:33:16.238 INFO:tasks.ceph.cluster2.osd.1:Started 2024-09-14T08:33:16.239 INFO:tasks.ceph.cluster2.osd.2:Restarting daemon 2024-09-14T08:33:16.239 DEBUG:teuthology.orchestra.run.smithi164:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster2 -i 2 2024-09-14T08:33:16.241 INFO:tasks.ceph.cluster2.osd.2:Started 2024-09-14T08:33:16.241 DEBUG:teuthology.orchestra.run.smithi164:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd dump --format=json 2024-09-14T08:33:16.413 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:33:16.413 INFO:teuthology.orchestra.run.smithi164.stdout:{"epoch":5,"fsid":"dd8fb0de-ee38-4627-a696-b631746d355c","created":"2024-09-14T08:33:15.389804+0000","modified":"2024-09-14T08:33:16.224104+0000","last_up_change":"0.000000","last_in_change":"2024-09-14T08:33:16.224104+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"93965f62-149b-4a3a-a65f-78058a6342df","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"dec8040a-7518-4bb9-84b5-0340c673c6bf","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"beca2c1e-834d-4485-b774-ece1dcd06a8d","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-14T08:33:16.424 INFO:tasks.ceph.ceph_manager.cluster2:[] 2024-09-14T08:33:16.424 INFO:tasks.ceph:Waiting for OSDs to come up 2024-09-14T08:33:16.727 DEBUG:teuthology.orchestra.run.smithi164:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster2 osd dump --format=json 2024-09-14T08:33:16.884 INFO:teuthology.misc.health.smithi164.stdout: 2024-09-14T08:33:16.884 INFO:teuthology.misc.health.smithi164.stdout:{"epoch":5,"fsid":"dd8fb0de-ee38-4627-a696-b631746d355c","created":"2024-09-14T08:33:15.389804+0000","modified":"2024-09-14T08:33:16.224104+0000","last_up_change":"0.000000","last_in_change":"2024-09-14T08:33:16.224104+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"93965f62-149b-4a3a-a65f-78058a6342df","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"dec8040a-7518-4bb9-84b5-0340c673c6bf","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"beca2c1e-834d-4485-b774-ece1dcd06a8d","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-14T08:33:16.893 DEBUG:teuthology.misc:0 of 3 OSDs are up 2024-09-14T08:33:17.675 INFO:tasks.ceph.cluster2.osd.2.smithi164.stderr:2024-09-14T08:33:17.675+0000 7fe3337bf640 -1 Falling back to public interface 2024-09-14T08:33:17.698 INFO:tasks.ceph.cluster2.osd.1.smithi164.stderr:2024-09-14T08:33:17.698+0000 7f90b9c0f640 -1 Falling back to public interface 2024-09-14T08:33:18.146 INFO:tasks.ceph.cluster2.osd.0.smithi164.stderr:2024-09-14T08:33:18.146+0000 7f7cf7eb4640 -1 Falling back to public interface 2024-09-14T08:33:19.792 INFO:tasks.ceph.cluster2.mgr.x.smithi135.stderr:2024-09-14T08:33:19.793+0000 7fc2f21f4640 -1 mgr.server handle_report got status from non-daemon mon.a 2024-09-14T08:33:21.197 INFO:tasks.ceph.cluster2.osd.2.smithi164.stderr:2024-09-14T08:33:21.197+0000 7fe3337bf640 -1 osd.2 0 log_to_monitors true 2024-09-14T08:33:21.487 INFO:tasks.ceph.cluster2.osd.1.smithi164.stderr:2024-09-14T08:33:21.487+0000 7f90b9c0f640 -1 osd.1 0 log_to_monitors true 2024-09-14T08:33:21.887 INFO:tasks.ceph.cluster2.osd.0.smithi164.stderr:2024-09-14T08:33:21.887+0000 7f7cf7eb4640 -1 osd.0 0 log_to_monitors true 2024-09-14T08:33:23.196 DEBUG:teuthology.orchestra.run.smithi164:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster2 osd dump --format=json 2024-09-14T08:33:23.487 INFO:teuthology.misc.health.smithi164.stdout: 2024-09-14T08:33:23.487 INFO:teuthology.misc.health.smithi164.stdout:{"epoch":7,"fsid":"dd8fb0de-ee38-4627-a696-b631746d355c","created":"2024-09-14T08:33:15.389804+0000","modified":"2024-09-14T08:33:22.797463+0000","last_up_change":"0.000000","last_in_change":"2024-09-14T08:33:16.224104+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"93965f62-149b-4a3a-a65f-78058a6342df","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"dec8040a-7518-4bb9-84b5-0340c673c6bf","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"beca2c1e-834d-4485-b774-ece1dcd06a8d","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-14T08:33:23.502 DEBUG:teuthology.misc:0 of 3 OSDs are up 2024-09-14T08:33:25.842 INFO:tasks.ceph.cluster2.osd.1.smithi164.stderr:2024-09-14T08:33:25.842+0000 7f90b5486640 -1 osd.1 0 waiting for initial osdmap 2024-09-14T08:33:25.851 INFO:tasks.ceph.cluster2.osd.1.smithi164.stderr:2024-09-14T08:33:25.852+0000 7f90ad2a5640 -1 osd.1 8 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-14T08:33:25.869 INFO:tasks.ceph.cluster2.osd.2.smithi164.stderr:2024-09-14T08:33:25.870+0000 7fe32f74e640 -1 osd.2 0 waiting for initial osdmap 2024-09-14T08:33:25.886 INFO:tasks.ceph.cluster2.osd.2.smithi164.stderr:2024-09-14T08:33:25.887+0000 7fe32756d640 -1 osd.2 8 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-14T08:33:26.760 INFO:tasks.ceph.cluster2.osd.0.smithi164.stderr:2024-09-14T08:33:26.760+0000 7f7cf4657640 -1 osd.0 0 waiting for initial osdmap 2024-09-14T08:33:26.775 INFO:tasks.ceph.cluster2.osd.0.smithi164.stderr:2024-09-14T08:33:26.776+0000 7f7cebc62640 -1 osd.0 8 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-14T08:33:29.805 DEBUG:teuthology.orchestra.run.smithi164:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster2 osd dump --format=json 2024-09-14T08:33:30.119 INFO:teuthology.misc.health.smithi164.stdout: 2024-09-14T08:33:30.120 INFO:teuthology.misc.health.smithi164.stdout:{"epoch":12,"fsid":"dd8fb0de-ee38-4627-a696-b631746d355c","created":"2024-09-14T08:33:15.389804+0000","modified":"2024-09-14T08:33:29.851651+0000","last_up_change":"2024-09-14T08:33:26.844101+0000","last_in_change":"2024-09-14T08:33:16.224104+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-14T08:33:28.808139+0000","flags":32769,"flags_names":"hashpspool,creating","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"93965f62-149b-4a3a-a65f-78058a6342df","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6816","nonce":2314553795},{"type":"v1","addr":"172.21.15.164:6817","nonce":2314553795}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6818","nonce":2314553795},{"type":"v1","addr":"172.21.15.164:6819","nonce":2314553795}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6822","nonce":2314553795},{"type":"v1","addr":"172.21.15.164:6823","nonce":2314553795}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6820","nonce":2314553795},{"type":"v1","addr":"172.21.15.164:6821","nonce":2314553795}]},"public_addr":"172.21.15.164:6817/2314553795","cluster_addr":"172.21.15.164:6819/2314553795","heartbeat_back_addr":"172.21.15.164:6823/2314553795","heartbeat_front_addr":"172.21.15.164:6821/2314553795","state":["exists","up"]},{"osd":1,"uuid":"dec8040a-7518-4bb9-84b5-0340c673c6bf","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":11,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6808","nonce":3507268573},{"type":"v1","addr":"172.21.15.164:6809","nonce":3507268573}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6810","nonce":3507268573},{"type":"v1","addr":"172.21.15.164:6811","nonce":3507268573}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6814","nonce":3507268573},{"type":"v1","addr":"172.21.15.164:6815","nonce":3507268573}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6812","nonce":3507268573},{"type":"v1","addr":"172.21.15.164:6813","nonce":3507268573}]},"public_addr":"172.21.15.164:6809/3507268573","cluster_addr":"172.21.15.164:6811/3507268573","heartbeat_back_addr":"172.21.15.164:6815/3507268573","heartbeat_front_addr":"172.21.15.164:6813/3507268573","state":["exists","up"]},{"osd":2,"uuid":"beca2c1e-834d-4485-b774-ece1dcd06a8d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6800","nonce":2722316831},{"type":"v1","addr":"172.21.15.164:6801","nonce":2722316831}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6802","nonce":2722316831},{"type":"v1","addr":"172.21.15.164:6803","nonce":2722316831}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6806","nonce":2722316831},{"type":"v1","addr":"172.21.15.164:6807","nonce":2722316831}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6804","nonce":2722316831},{"type":"v1","addr":"172.21.15.164:6805","nonce":2722316831}]},"public_addr":"172.21.15.164:6801/2722316831","cluster_addr":"172.21.15.164:6803/2722316831","heartbeat_back_addr":"172.21.15.164:6807/2722316831","heartbeat_front_addr":"172.21.15.164:6805/2722316831","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T08:33:22.868388+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T08:33:22.449398+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T08:33:22.240282+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-14T08:33:30.133 DEBUG:teuthology.misc:3 of 3 OSDs are up 2024-09-14T08:33:30.133 INFO:tasks.ceph:Creating RBD pool 2024-09-14T08:33:30.134 DEBUG:teuthology.orchestra.run.smithi164:> sudo ceph --cluster cluster2 osd pool create rbd 8 2024-09-14T08:33:30.859 INFO:teuthology.orchestra.run.smithi164.stderr:pool 'rbd' created 2024-09-14T08:33:30.875 DEBUG:teuthology.orchestra.run.smithi164:> rbd --cluster cluster2 pool init rbd 2024-09-14T08:33:33.874 INFO:tasks.ceph:Starting mds daemons in cluster cluster2... 2024-09-14T08:33:33.875 DEBUG:teuthology.orchestra.run.smithi164:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 config log 1 --format=json 2024-09-14T08:33:33.875 INFO:tasks.daemonwatchdog.daemon_watchdog:watchdog starting 2024-09-14T08:33:34.170 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:33:34.182 INFO:teuthology.orchestra.run.smithi164.stdout:[{"version":4,"timestamp":"2024-09-14T08:33:26.761780+0000","name":"","changes":[{"name":"osd.0/osd_mclock_max_capacity_iops_ssd","new_value":"6783.878060"}]}] 2024-09-14T08:33:34.182 INFO:tasks.ceph_manager:config epoch is 4 2024-09-14T08:33:34.183 INFO:tasks.ceph:Waiting until cluster2 daemons up and pgs clean... 2024-09-14T08:33:34.183 INFO:tasks.ceph.ceph_manager.cluster2:waiting for mgr available 2024-09-14T08:33:34.183 DEBUG:teuthology.orchestra.run.smithi164:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 mgr dump --format=json 2024-09-14T08:33:34.515 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:33:34.525 INFO:teuthology.orchestra.run.smithi164.stdout:{"epoch":5,"flags":0,"active_gid":4098,"active_name":"x","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6826","nonce":614612851},{"type":"v1","addr":"172.21.15.135:6827","nonce":614612851}]},"active_addr":"172.21.15.135:6827/614612851","active_change":"2024-09-14T08:33:18.785700+0000","active_mgr_features":4540719139924082687,"available":true,"standbys":[],"modules":["iostat","nfs","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to, use commas to separate multiple","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.27.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:10.4.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_nginx":{"name":"container_image_nginx","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nginx:1.26.1","min":"","max":"","enum_allowed":[],"desc":"Nginx container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.7.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.2.17","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_oauth2_proxy":{"name":"container_image_oauth2_proxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/oauth2-proxy/oauth2-proxy:v7.6.0","min":"","max":"","enum_allowed":[],"desc":"oauth2-proxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.51.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba":{"name":"container_image_samba","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-server:devbuilds-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba/SMB container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MANAGED_BY_CLUSTERS":{"name":"MANAGED_BY_CLUSTERS","type":"str","level":"advanced","flags":0,"default_value":"[]","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MULTICLUSTER_CONFIG":{"name":"MULTICLUSTER_CONFIG","type":"str","level":"advanced","flags":0,"default_value":"{}","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_requests":{"name":"max_requests","type":"int","level":"advanced","flags":0,"default_value":"500","min":"","max":"","enum_allowed":[],"desc":"Maximum number of requests to keep in memory. When new request comes in, the oldest request will be removed if the number of requests exceeds the max request number.if un-finished request is removed, error message will be logged in the ceph-mgr log.","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"secondary_zone_period_retry_limit":{"name":"secondary_zone_period_retry_limit","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"RGW module period update retry limit for secondary site","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"smb","can_run":true,"error_string":"","module_options":{"internal_store_backend":{"name":"internal_store_backend","type":"str","level":"dev","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"set internal store backend. for develoment and testing only","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"update_orchestration":{"name":"update_orchestration","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically update orchestration when smb resources are changed","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"squid":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":0,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.135:0","nonce":234347820}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.135:0","nonce":4113029076}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.135:0","nonce":3307875335}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.135:0","nonce":2307870167}]}]} 2024-09-14T08:33:34.529 INFO:tasks.ceph.ceph_manager.cluster2:mgr available! 2024-09-14T08:33:34.529 INFO:tasks.ceph.ceph_manager.cluster2:waiting for all up 2024-09-14T08:33:34.530 DEBUG:teuthology.orchestra.run.smithi164:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd dump --format=json 2024-09-14T08:33:34.814 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:33:34.815 INFO:teuthology.orchestra.run.smithi164.stdout:{"epoch":16,"fsid":"dd8fb0de-ee38-4627-a696-b631746d355c","created":"2024-09-14T08:33:15.389804+0000","modified":"2024-09-14T08:33:33.862729+0000","last_up_change":"2024-09-14T08:33:26.844101+0000","last_in_change":"2024-09-14T08:33:16.224104+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-14T08:33:28.808139+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-14T08:33:30.421146+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":16,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"93965f62-149b-4a3a-a65f-78058a6342df","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6816","nonce":2314553795},{"type":"v1","addr":"172.21.15.164:6817","nonce":2314553795}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6818","nonce":2314553795},{"type":"v1","addr":"172.21.15.164:6819","nonce":2314553795}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6822","nonce":2314553795},{"type":"v1","addr":"172.21.15.164:6823","nonce":2314553795}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6820","nonce":2314553795},{"type":"v1","addr":"172.21.15.164:6821","nonce":2314553795}]},"public_addr":"172.21.15.164:6817/2314553795","cluster_addr":"172.21.15.164:6819/2314553795","heartbeat_back_addr":"172.21.15.164:6823/2314553795","heartbeat_front_addr":"172.21.15.164:6821/2314553795","state":["exists","up"]},{"osd":1,"uuid":"dec8040a-7518-4bb9-84b5-0340c673c6bf","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6808","nonce":3507268573},{"type":"v1","addr":"172.21.15.164:6809","nonce":3507268573}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6810","nonce":3507268573},{"type":"v1","addr":"172.21.15.164:6811","nonce":3507268573}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6814","nonce":3507268573},{"type":"v1","addr":"172.21.15.164:6815","nonce":3507268573}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6812","nonce":3507268573},{"type":"v1","addr":"172.21.15.164:6813","nonce":3507268573}]},"public_addr":"172.21.15.164:6809/3507268573","cluster_addr":"172.21.15.164:6811/3507268573","heartbeat_back_addr":"172.21.15.164:6815/3507268573","heartbeat_front_addr":"172.21.15.164:6813/3507268573","state":["exists","up"]},{"osd":2,"uuid":"beca2c1e-834d-4485-b774-ece1dcd06a8d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6800","nonce":2722316831},{"type":"v1","addr":"172.21.15.164:6801","nonce":2722316831}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6802","nonce":2722316831},{"type":"v1","addr":"172.21.15.164:6803","nonce":2722316831}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6806","nonce":2722316831},{"type":"v1","addr":"172.21.15.164:6807","nonce":2722316831}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6804","nonce":2722316831},{"type":"v1","addr":"172.21.15.164:6805","nonce":2722316831}]},"public_addr":"172.21.15.164:6801/2722316831","cluster_addr":"172.21.15.164:6803/2722316831","heartbeat_back_addr":"172.21.15.164:6807/2722316831","heartbeat_front_addr":"172.21.15.164:6805/2722316831","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T08:33:22.868388+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T08:33:22.449398+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T08:33:22.240282+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-14T08:33:34.827 INFO:tasks.ceph.ceph_manager.cluster2:all up! 2024-09-14T08:33:34.827 DEBUG:teuthology.orchestra.run.smithi164:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd dump --format=json 2024-09-14T08:33:35.110 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:33:35.110 INFO:teuthology.orchestra.run.smithi164.stdout:{"epoch":16,"fsid":"dd8fb0de-ee38-4627-a696-b631746d355c","created":"2024-09-14T08:33:15.389804+0000","modified":"2024-09-14T08:33:33.862729+0000","last_up_change":"2024-09-14T08:33:26.844101+0000","last_in_change":"2024-09-14T08:33:16.224104+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-14T08:33:28.808139+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-14T08:33:30.421146+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":16,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"93965f62-149b-4a3a-a65f-78058a6342df","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6816","nonce":2314553795},{"type":"v1","addr":"172.21.15.164:6817","nonce":2314553795}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6818","nonce":2314553795},{"type":"v1","addr":"172.21.15.164:6819","nonce":2314553795}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6822","nonce":2314553795},{"type":"v1","addr":"172.21.15.164:6823","nonce":2314553795}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6820","nonce":2314553795},{"type":"v1","addr":"172.21.15.164:6821","nonce":2314553795}]},"public_addr":"172.21.15.164:6817/2314553795","cluster_addr":"172.21.15.164:6819/2314553795","heartbeat_back_addr":"172.21.15.164:6823/2314553795","heartbeat_front_addr":"172.21.15.164:6821/2314553795","state":["exists","up"]},{"osd":1,"uuid":"dec8040a-7518-4bb9-84b5-0340c673c6bf","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6808","nonce":3507268573},{"type":"v1","addr":"172.21.15.164:6809","nonce":3507268573}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6810","nonce":3507268573},{"type":"v1","addr":"172.21.15.164:6811","nonce":3507268573}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6814","nonce":3507268573},{"type":"v1","addr":"172.21.15.164:6815","nonce":3507268573}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6812","nonce":3507268573},{"type":"v1","addr":"172.21.15.164:6813","nonce":3507268573}]},"public_addr":"172.21.15.164:6809/3507268573","cluster_addr":"172.21.15.164:6811/3507268573","heartbeat_back_addr":"172.21.15.164:6815/3507268573","heartbeat_front_addr":"172.21.15.164:6813/3507268573","state":["exists","up"]},{"osd":2,"uuid":"beca2c1e-834d-4485-b774-ece1dcd06a8d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6800","nonce":2722316831},{"type":"v1","addr":"172.21.15.164:6801","nonce":2722316831}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6802","nonce":2722316831},{"type":"v1","addr":"172.21.15.164:6803","nonce":2722316831}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6806","nonce":2722316831},{"type":"v1","addr":"172.21.15.164:6807","nonce":2722316831}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6804","nonce":2722316831},{"type":"v1","addr":"172.21.15.164:6805","nonce":2722316831}]},"public_addr":"172.21.15.164:6801/2722316831","cluster_addr":"172.21.15.164:6803/2722316831","heartbeat_back_addr":"172.21.15.164:6807/2722316831","heartbeat_front_addr":"172.21.15.164:6805/2722316831","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T08:33:22.868388+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T08:33:22.449398+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T08:33:22.240282+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-14T08:33:35.123 DEBUG:teuthology.orchestra.run.smithi164:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.0 flush_pg_stats 2024-09-14T08:33:35.124 DEBUG:teuthology.orchestra.run.smithi164:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.1 flush_pg_stats 2024-09-14T08:33:35.124 DEBUG:teuthology.orchestra.run.smithi164:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.2 flush_pg_stats 2024-09-14T08:33:35.262 INFO:teuthology.orchestra.run.smithi164.stdout:38654705671 2024-09-14T08:33:35.263 DEBUG:teuthology.orchestra.run.smithi164:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.1 2024-09-14T08:33:35.264 INFO:teuthology.orchestra.run.smithi164.stdout:38654705672 2024-09-14T08:33:35.264 DEBUG:teuthology.orchestra.run.smithi164:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.2 2024-09-14T08:33:35.265 INFO:teuthology.orchestra.run.smithi164.stdout:38654705671 2024-09-14T08:33:35.265 DEBUG:teuthology.orchestra.run.smithi164:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.0 2024-09-14T08:33:35.608 INFO:teuthology.orchestra.run.smithi164.stdout:38654705670 2024-09-14T08:33:35.621 INFO:tasks.ceph.ceph_manager.cluster2:need seq 38654705671 got 38654705670 for osd.1 2024-09-14T08:33:35.625 INFO:teuthology.orchestra.run.smithi164.stdout:38654705671 2024-09-14T08:33:35.639 INFO:tasks.ceph.ceph_manager.cluster2:need seq 38654705672 got 38654705671 for osd.2 2024-09-14T08:33:35.639 INFO:teuthology.orchestra.run.smithi164.stdout:38654705670 2024-09-14T08:33:35.653 INFO:tasks.ceph.ceph_manager.cluster2:need seq 38654705671 got 38654705670 for osd.0 2024-09-14T08:33:36.622 DEBUG:teuthology.orchestra.run.smithi164:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.1 2024-09-14T08:33:36.639 DEBUG:teuthology.orchestra.run.smithi164:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.2 2024-09-14T08:33:36.653 DEBUG:teuthology.orchestra.run.smithi164:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.0 2024-09-14T08:33:36.972 INFO:teuthology.orchestra.run.smithi164.stdout:38654705671 2024-09-14T08:33:36.986 INFO:tasks.ceph.ceph_manager.cluster2:need seq 38654705671 got 38654705671 for osd.1 2024-09-14T08:33:36.986 INFO:teuthology.orchestra.run.smithi164.stdout:38654705672 2024-09-14T08:33:36.986 DEBUG:teuthology.parallel:result is None 2024-09-14T08:33:36.998 INFO:tasks.ceph.ceph_manager.cluster2:need seq 38654705672 got 38654705672 for osd.2 2024-09-14T08:33:36.999 DEBUG:teuthology.parallel:result is None 2024-09-14T08:33:37.000 INFO:teuthology.orchestra.run.smithi164.stdout:38654705671 2024-09-14T08:33:37.012 INFO:tasks.ceph.ceph_manager.cluster2:need seq 38654705671 got 38654705671 for osd.0 2024-09-14T08:33:37.013 DEBUG:teuthology.parallel:result is None 2024-09-14T08:33:37.013 INFO:tasks.ceph.ceph_manager.cluster2:waiting for clean 2024-09-14T08:33:37.013 DEBUG:teuthology.orchestra.run.smithi164:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2024-09-14T08:33:37.332 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:33:37.332 INFO:teuthology.orchestra.run.smithi164.stderr:dumped all 2024-09-14T08:33:37.344 INFO:teuthology.orchestra.run.smithi164.stdout:{"pg_ready":true,"pg_map":{"version":19,"stamp":"2024-09-14T08:33:36.792865+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":81,"num_write_kb":1108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":81640,"kb_used_data":528,"kb_used_omap":4,"kb_used_meta":81019,"kb_avail":283033880,"statfs":{"total":289910292480,"available":289826693120,"internally_reserved":0,"allocated":540672,"data_stored":1301215,"data_compressed":7088,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":4785,"internal_metadata":82963791},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"3.929519"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":20,"reported_epoch":16,"state":"active+clean","last_fresh":"2024-09-14T08:33:33.870261+0000","last_change":"2024-09-14T08:33:33.870517+0000","last_active":"2024-09-14T08:33:33.870261+0000","last_peered":"2024-09-14T08:33:33.870261+0000","last_clean":"2024-09-14T08:33:33.870261+0000","last_became_active":"2024-09-14T08:33:31.868938+0000","last_became_peered":"2024-09-14T08:33:31.868938+0000","last_unstale":"2024-09-14T08:33:33.870261+0000","last_undegraded":"2024-09-14T08:33:33.870261+0000","last_fullsized":"2024-09-14T08:33:33.870261+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:33:30.860386+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:33:30.860386+0000","last_clean_scrub_stamp":"2024-09-14T08:33:30.860386+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:25:40.377777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00071425700000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.6","version":"0'0","reported_seq":20,"reported_epoch":16,"state":"active+clean","last_fresh":"2024-09-14T08:33:33.870646+0000","last_change":"2024-09-14T08:33:33.870929+0000","last_active":"2024-09-14T08:33:33.870646+0000","last_peered":"2024-09-14T08:33:33.870646+0000","last_clean":"2024-09-14T08:33:33.870646+0000","last_became_active":"2024-09-14T08:33:31.868362+0000","last_became_peered":"2024-09-14T08:33:31.868362+0000","last_unstale":"2024-09-14T08:33:33.870646+0000","last_undegraded":"2024-09-14T08:33:33.870646+0000","last_fullsized":"2024-09-14T08:33:33.870646+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:33:30.860386+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:33:30.860386+0000","last_clean_scrub_stamp":"2024-09-14T08:33:30.860386+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:59:01.028926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001256898,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.5","version":"0'0","reported_seq":20,"reported_epoch":16,"state":"active+clean","last_fresh":"2024-09-14T08:33:33.870327+0000","last_change":"2024-09-14T08:33:33.870538+0000","last_active":"2024-09-14T08:33:33.870327+0000","last_peered":"2024-09-14T08:33:33.870327+0000","last_clean":"2024-09-14T08:33:33.870327+0000","last_became_active":"2024-09-14T08:33:31.868180+0000","last_became_peered":"2024-09-14T08:33:31.868180+0000","last_unstale":"2024-09-14T08:33:33.870327+0000","last_undegraded":"2024-09-14T08:33:33.870327+0000","last_fullsized":"2024-09-14T08:33:33.870327+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:33:30.860386+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:33:30.860386+0000","last_clean_scrub_stamp":"2024-09-14T08:33:30.860386+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T19:51:08.921688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00044658000000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.4","version":"0'0","reported_seq":20,"reported_epoch":16,"state":"active+clean","last_fresh":"2024-09-14T08:33:33.870586+0000","last_change":"2024-09-14T08:33:33.870794+0000","last_active":"2024-09-14T08:33:33.870586+0000","last_peered":"2024-09-14T08:33:33.870586+0000","last_clean":"2024-09-14T08:33:33.870586+0000","last_became_active":"2024-09-14T08:33:31.868126+0000","last_became_peered":"2024-09-14T08:33:31.868126+0000","last_unstale":"2024-09-14T08:33:33.870586+0000","last_undegraded":"2024-09-14T08:33:33.870586+0000","last_fullsized":"2024-09-14T08:33:33.870586+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:33:30.860386+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:33:30.860386+0000","last_clean_scrub_stamp":"2024-09-14T08:33:30.860386+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:20:30.169758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064856099999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.2","version":"16'2","reported_seq":22,"reported_epoch":16,"state":"active+clean","last_fresh":"2024-09-14T08:33:33.865752+0000","last_change":"2024-09-14T08:33:33.865752+0000","last_active":"2024-09-14T08:33:33.865752+0000","last_peered":"2024-09-14T08:33:33.865752+0000","last_clean":"2024-09-14T08:33:33.865752+0000","last_became_active":"2024-09-14T08:33:31.870304+0000","last_became_peered":"2024-09-14T08:33:31.870304+0000","last_unstale":"2024-09-14T08:33:33.865752+0000","last_undegraded":"2024-09-14T08:33:33.865752+0000","last_fullsized":"2024-09-14T08:33:33.865752+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:33:30.860386+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:33:30.860386+0000","last_clean_scrub_stamp":"2024-09-14T08:33:30.860386+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:17:46.077907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000475141,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.1","version":"0'0","reported_seq":20,"reported_epoch":16,"state":"active+clean","last_fresh":"2024-09-14T08:33:34.137343+0000","last_change":"2024-09-14T08:33:34.137500+0000","last_active":"2024-09-14T08:33:34.137343+0000","last_peered":"2024-09-14T08:33:34.137343+0000","last_clean":"2024-09-14T08:33:34.137343+0000","last_became_active":"2024-09-14T08:33:31.869506+0000","last_became_peered":"2024-09-14T08:33:31.869506+0000","last_unstale":"2024-09-14T08:33:34.137343+0000","last_undegraded":"2024-09-14T08:33:34.137343+0000","last_fullsized":"2024-09-14T08:33:34.137343+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:33:30.860386+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:33:30.860386+0000","last_clean_scrub_stamp":"2024-09-14T08:33:30.860386+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:37:11.985802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036617900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.0","version":"0'0","reported_seq":20,"reported_epoch":16,"state":"active+clean","last_fresh":"2024-09-14T08:33:34.137397+0000","last_change":"2024-09-14T08:33:34.137555+0000","last_active":"2024-09-14T08:33:34.137397+0000","last_peered":"2024-09-14T08:33:34.137397+0000","last_clean":"2024-09-14T08:33:34.137397+0000","last_became_active":"2024-09-14T08:33:31.868768+0000","last_became_peered":"2024-09-14T08:33:31.868768+0000","last_unstale":"2024-09-14T08:33:34.137397+0000","last_undegraded":"2024-09-14T08:33:34.137397+0000","last_fullsized":"2024-09-14T08:33:34.137397+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:33:30.860386+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:33:30.860386+0000","last_clean_scrub_stamp":"2024-09-14T08:33:30.860386+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:16:44.486807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041116299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.3","version":"14'1","reported_seq":21,"reported_epoch":16,"state":"active+clean","last_fresh":"2024-09-14T08:33:33.869896+0000","last_change":"2024-09-14T08:33:33.870202+0000","last_active":"2024-09-14T08:33:33.869896+0000","last_peered":"2024-09-14T08:33:33.869896+0000","last_clean":"2024-09-14T08:33:33.869896+0000","last_became_active":"2024-09-14T08:33:31.868308+0000","last_became_peered":"2024-09-14T08:33:31.868308+0000","last_unstale":"2024-09-14T08:33:33.869896+0000","last_undegraded":"2024-09-14T08:33:33.869896+0000","last_fullsized":"2024-09-14T08:33:33.869896+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:33:30.860386+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:33:30.860386+0000","last_clean_scrub_stamp":"2024-09-14T08:33:30.860386+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T17:51:35.908288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00058585600000000005,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"1.0","version":"12'52","reported_seq":86,"reported_epoch":16,"state":"active+clean","last_fresh":"2024-09-14T08:33:33.869548+0000","last_change":"2024-09-14T08:33:29.863094+0000","last_active":"2024-09-14T08:33:33.869548+0000","last_peered":"2024-09-14T08:33:33.869548+0000","last_clean":"2024-09-14T08:33:33.869548+0000","last_became_active":"2024-09-14T08:33:29.862802+0000","last_became_peered":"2024-09-14T08:33:29.862802+0000","last_unstale":"2024-09-14T08:33:33.869548+0000","last_undegraded":"2024-09-14T08:33:33.869548+0000","last_fullsized":"2024-09-14T08:33:33.869548+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:33:28.845692+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:33:28.845692+0000","last_clean_scrub_stamp":"2024-09-14T08:33:28.845692+0000","objects_scrubbed":0,"log_size":52,"log_dups_size":0,"ondisk_log_size":52,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:35:27.535389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":79,"num_write_kb":1106,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":79,"num_write_kb":1106,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":1180736,"data_compressed":7088,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":52,"ondisk_log_size":52,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654705672,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27164,"kb_used_data":136,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344676,"statfs":{"total":96636764160,"available":96608948224,"internally_reserved":0,"allocated":139264,"data_stored":40147,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":9,"seq":38654705671,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27224,"kb_used_data":196,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344616,"statfs":{"total":96636764160,"available":96608886784,"internally_reserved":0,"allocated":200704,"data_stored":630534,"data_compressed":3544,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":9,"seq":38654705671,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27252,"kb_used_data":196,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344588,"statfs":{"total":96636764160,"available":96608858112,"internally_reserved":0,"allocated":200704,"data_stored":630534,"data_compressed":3544,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":3544,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":3544,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T08:33:37.345 DEBUG:teuthology.orchestra.run.smithi164:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2024-09-14T08:33:37.625 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:33:37.625 INFO:teuthology.orchestra.run.smithi164.stderr:dumped all 2024-09-14T08:33:37.637 INFO:teuthology.orchestra.run.smithi164.stdout:{"pg_ready":true,"pg_map":{"version":19,"stamp":"2024-09-14T08:33:36.792865+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":81,"num_write_kb":1108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":81640,"kb_used_data":528,"kb_used_omap":4,"kb_used_meta":81019,"kb_avail":283033880,"statfs":{"total":289910292480,"available":289826693120,"internally_reserved":0,"allocated":540672,"data_stored":1301215,"data_compressed":7088,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":4785,"internal_metadata":82963791},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"3.929519"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":20,"reported_epoch":16,"state":"active+clean","last_fresh":"2024-09-14T08:33:33.870261+0000","last_change":"2024-09-14T08:33:33.870517+0000","last_active":"2024-09-14T08:33:33.870261+0000","last_peered":"2024-09-14T08:33:33.870261+0000","last_clean":"2024-09-14T08:33:33.870261+0000","last_became_active":"2024-09-14T08:33:31.868938+0000","last_became_peered":"2024-09-14T08:33:31.868938+0000","last_unstale":"2024-09-14T08:33:33.870261+0000","last_undegraded":"2024-09-14T08:33:33.870261+0000","last_fullsized":"2024-09-14T08:33:33.870261+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:33:30.860386+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:33:30.860386+0000","last_clean_scrub_stamp":"2024-09-14T08:33:30.860386+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:25:40.377777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00071425700000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.6","version":"0'0","reported_seq":20,"reported_epoch":16,"state":"active+clean","last_fresh":"2024-09-14T08:33:33.870646+0000","last_change":"2024-09-14T08:33:33.870929+0000","last_active":"2024-09-14T08:33:33.870646+0000","last_peered":"2024-09-14T08:33:33.870646+0000","last_clean":"2024-09-14T08:33:33.870646+0000","last_became_active":"2024-09-14T08:33:31.868362+0000","last_became_peered":"2024-09-14T08:33:31.868362+0000","last_unstale":"2024-09-14T08:33:33.870646+0000","last_undegraded":"2024-09-14T08:33:33.870646+0000","last_fullsized":"2024-09-14T08:33:33.870646+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:33:30.860386+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:33:30.860386+0000","last_clean_scrub_stamp":"2024-09-14T08:33:30.860386+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:59:01.028926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001256898,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.5","version":"0'0","reported_seq":20,"reported_epoch":16,"state":"active+clean","last_fresh":"2024-09-14T08:33:33.870327+0000","last_change":"2024-09-14T08:33:33.870538+0000","last_active":"2024-09-14T08:33:33.870327+0000","last_peered":"2024-09-14T08:33:33.870327+0000","last_clean":"2024-09-14T08:33:33.870327+0000","last_became_active":"2024-09-14T08:33:31.868180+0000","last_became_peered":"2024-09-14T08:33:31.868180+0000","last_unstale":"2024-09-14T08:33:33.870327+0000","last_undegraded":"2024-09-14T08:33:33.870327+0000","last_fullsized":"2024-09-14T08:33:33.870327+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:33:30.860386+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:33:30.860386+0000","last_clean_scrub_stamp":"2024-09-14T08:33:30.860386+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T19:51:08.921688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00044658000000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.4","version":"0'0","reported_seq":20,"reported_epoch":16,"state":"active+clean","last_fresh":"2024-09-14T08:33:33.870586+0000","last_change":"2024-09-14T08:33:33.870794+0000","last_active":"2024-09-14T08:33:33.870586+0000","last_peered":"2024-09-14T08:33:33.870586+0000","last_clean":"2024-09-14T08:33:33.870586+0000","last_became_active":"2024-09-14T08:33:31.868126+0000","last_became_peered":"2024-09-14T08:33:31.868126+0000","last_unstale":"2024-09-14T08:33:33.870586+0000","last_undegraded":"2024-09-14T08:33:33.870586+0000","last_fullsized":"2024-09-14T08:33:33.870586+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:33:30.860386+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:33:30.860386+0000","last_clean_scrub_stamp":"2024-09-14T08:33:30.860386+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:20:30.169758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064856099999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.2","version":"16'2","reported_seq":22,"reported_epoch":16,"state":"active+clean","last_fresh":"2024-09-14T08:33:33.865752+0000","last_change":"2024-09-14T08:33:33.865752+0000","last_active":"2024-09-14T08:33:33.865752+0000","last_peered":"2024-09-14T08:33:33.865752+0000","last_clean":"2024-09-14T08:33:33.865752+0000","last_became_active":"2024-09-14T08:33:31.870304+0000","last_became_peered":"2024-09-14T08:33:31.870304+0000","last_unstale":"2024-09-14T08:33:33.865752+0000","last_undegraded":"2024-09-14T08:33:33.865752+0000","last_fullsized":"2024-09-14T08:33:33.865752+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:33:30.860386+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:33:30.860386+0000","last_clean_scrub_stamp":"2024-09-14T08:33:30.860386+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:17:46.077907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000475141,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.1","version":"0'0","reported_seq":20,"reported_epoch":16,"state":"active+clean","last_fresh":"2024-09-14T08:33:34.137343+0000","last_change":"2024-09-14T08:33:34.137500+0000","last_active":"2024-09-14T08:33:34.137343+0000","last_peered":"2024-09-14T08:33:34.137343+0000","last_clean":"2024-09-14T08:33:34.137343+0000","last_became_active":"2024-09-14T08:33:31.869506+0000","last_became_peered":"2024-09-14T08:33:31.869506+0000","last_unstale":"2024-09-14T08:33:34.137343+0000","last_undegraded":"2024-09-14T08:33:34.137343+0000","last_fullsized":"2024-09-14T08:33:34.137343+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:33:30.860386+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:33:30.860386+0000","last_clean_scrub_stamp":"2024-09-14T08:33:30.860386+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:37:11.985802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036617900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.0","version":"0'0","reported_seq":20,"reported_epoch":16,"state":"active+clean","last_fresh":"2024-09-14T08:33:34.137397+0000","last_change":"2024-09-14T08:33:34.137555+0000","last_active":"2024-09-14T08:33:34.137397+0000","last_peered":"2024-09-14T08:33:34.137397+0000","last_clean":"2024-09-14T08:33:34.137397+0000","last_became_active":"2024-09-14T08:33:31.868768+0000","last_became_peered":"2024-09-14T08:33:31.868768+0000","last_unstale":"2024-09-14T08:33:34.137397+0000","last_undegraded":"2024-09-14T08:33:34.137397+0000","last_fullsized":"2024-09-14T08:33:34.137397+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:33:30.860386+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:33:30.860386+0000","last_clean_scrub_stamp":"2024-09-14T08:33:30.860386+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:16:44.486807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041116299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.3","version":"14'1","reported_seq":21,"reported_epoch":16,"state":"active+clean","last_fresh":"2024-09-14T08:33:33.869896+0000","last_change":"2024-09-14T08:33:33.870202+0000","last_active":"2024-09-14T08:33:33.869896+0000","last_peered":"2024-09-14T08:33:33.869896+0000","last_clean":"2024-09-14T08:33:33.869896+0000","last_became_active":"2024-09-14T08:33:31.868308+0000","last_became_peered":"2024-09-14T08:33:31.868308+0000","last_unstale":"2024-09-14T08:33:33.869896+0000","last_undegraded":"2024-09-14T08:33:33.869896+0000","last_fullsized":"2024-09-14T08:33:33.869896+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:33:30.860386+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:33:30.860386+0000","last_clean_scrub_stamp":"2024-09-14T08:33:30.860386+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T17:51:35.908288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00058585600000000005,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"1.0","version":"12'52","reported_seq":86,"reported_epoch":16,"state":"active+clean","last_fresh":"2024-09-14T08:33:33.869548+0000","last_change":"2024-09-14T08:33:29.863094+0000","last_active":"2024-09-14T08:33:33.869548+0000","last_peered":"2024-09-14T08:33:33.869548+0000","last_clean":"2024-09-14T08:33:33.869548+0000","last_became_active":"2024-09-14T08:33:29.862802+0000","last_became_peered":"2024-09-14T08:33:29.862802+0000","last_unstale":"2024-09-14T08:33:33.869548+0000","last_undegraded":"2024-09-14T08:33:33.869548+0000","last_fullsized":"2024-09-14T08:33:33.869548+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:33:28.845692+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:33:28.845692+0000","last_clean_scrub_stamp":"2024-09-14T08:33:28.845692+0000","objects_scrubbed":0,"log_size":52,"log_dups_size":0,"ondisk_log_size":52,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:35:27.535389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":79,"num_write_kb":1106,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":79,"num_write_kb":1106,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":1180736,"data_compressed":7088,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":52,"ondisk_log_size":52,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654705672,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27164,"kb_used_data":136,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344676,"statfs":{"total":96636764160,"available":96608948224,"internally_reserved":0,"allocated":139264,"data_stored":40147,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":9,"seq":38654705671,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27224,"kb_used_data":196,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344616,"statfs":{"total":96636764160,"available":96608886784,"internally_reserved":0,"allocated":200704,"data_stored":630534,"data_compressed":3544,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":9,"seq":38654705671,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27252,"kb_used_data":196,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344588,"statfs":{"total":96636764160,"available":96608858112,"internally_reserved":0,"allocated":200704,"data_stored":630534,"data_compressed":3544,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":3544,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":3544,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T08:33:37.638 INFO:tasks.ceph.ceph_manager.cluster2:clean! 2024-09-14T08:33:37.638 INFO:tasks.ceph:Waiting until ceph cluster cluster2 is healthy... 2024-09-14T08:33:37.638 INFO:tasks.ceph.ceph_manager.cluster2:wait_until_healthy 2024-09-14T08:33:37.638 DEBUG:teuthology.orchestra.run.smithi164:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 health --format=json 2024-09-14T08:33:37.956 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T08:33:37.956 INFO:teuthology.orchestra.run.smithi164.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-09-14T08:33:37.969 INFO:tasks.ceph.ceph_manager.cluster2:wait_until_healthy done 2024-09-14T08:33:37.969 INFO:teuthology.run_tasks:Running task exec... 2024-09-14T08:33:37.978 INFO:teuthology.task.exec:Executing custom commands... 2024-09-14T08:33:37.978 INFO:teuthology.task.exec:Running commands on role cluster1.client.mirror.0 host ubuntu@smithi164.front.sepia.ceph.com 2024-09-14T08:33:37.978 DEBUG:teuthology.orchestra.run.smithi164:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror mon '"'"'profile rbd-mirror-peer'"'"' osd '"'"'profile rbd'"'"'' 2024-09-14T08:33:38.344 INFO:teuthology.orchestra.run.smithi164.stdout:[client.mirror] 2024-09-14T08:33:38.344 INFO:teuthology.orchestra.run.smithi164.stdout: key = AQAVSuVmNEp6NRAAd0nGGnnjv95u4Lypkcb1Sg== 2024-09-14T08:33:38.344 INFO:teuthology.orchestra.run.smithi164.stdout: caps mon = "profile rbd-mirror-peer" 2024-09-14T08:33:38.344 INFO:teuthology.orchestra.run.smithi164.stdout: caps osd = "profile rbd" 2024-09-14T08:33:38.344 INFO:teuthology.orchestra.run.smithi164.stderr:updated caps for client.mirror 2024-09-14T08:33:38.357 DEBUG:teuthology.orchestra.run.smithi164:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.0 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2024-09-14T08:33:38.718 INFO:teuthology.orchestra.run.smithi164.stdout:[client.mirror.0] 2024-09-14T08:33:38.718 INFO:teuthology.orchestra.run.smithi164.stdout: key = AQAVSuVmypPsOBAA5ZqBI4/dRwbC2kbumMna0Q== 2024-09-14T08:33:38.719 INFO:teuthology.orchestra.run.smithi164.stdout: caps mon = "profile rbd-mirror" 2024-09-14T08:33:38.719 INFO:teuthology.orchestra.run.smithi164.stdout: caps osd = "profile rbd" 2024-09-14T08:33:38.719 INFO:teuthology.orchestra.run.smithi164.stderr:updated caps for client.mirror.0 2024-09-14T08:33:38.731 DEBUG:teuthology.orchestra.run.smithi164:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.1 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2024-09-14T08:33:39.090 INFO:teuthology.orchestra.run.smithi164.stdout:[client.mirror.1] 2024-09-14T08:33:39.090 INFO:teuthology.orchestra.run.smithi164.stdout: key = AQAWSuVmpQn7AhAAw10ZlA8g9gGLtqer42Mv2g== 2024-09-14T08:33:39.090 INFO:teuthology.orchestra.run.smithi164.stdout: caps mon = "profile rbd-mirror" 2024-09-14T08:33:39.090 INFO:teuthology.orchestra.run.smithi164.stdout: caps osd = "profile rbd" 2024-09-14T08:33:39.091 INFO:teuthology.orchestra.run.smithi164.stderr:updated caps for client.mirror.1 2024-09-14T08:33:39.103 DEBUG:teuthology.orchestra.run.smithi164:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.2 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2024-09-14T08:33:39.468 INFO:teuthology.orchestra.run.smithi164.stdout:[client.mirror.2] 2024-09-14T08:33:39.468 INFO:teuthology.orchestra.run.smithi164.stdout: key = AQAWSuVmVNGVCBAAJeLJHUqgK83KJhmZFWVyjw== 2024-09-14T08:33:39.468 INFO:teuthology.orchestra.run.smithi164.stdout: caps mon = "profile rbd-mirror" 2024-09-14T08:33:39.468 INFO:teuthology.orchestra.run.smithi164.stdout: caps osd = "profile rbd" 2024-09-14T08:33:39.468 INFO:teuthology.orchestra.run.smithi164.stderr:updated caps for client.mirror.2 2024-09-14T08:33:39.481 DEBUG:teuthology.orchestra.run.smithi164:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.3 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2024-09-14T08:33:39.843 INFO:teuthology.orchestra.run.smithi164.stdout:[client.mirror.3] 2024-09-14T08:33:39.844 INFO:teuthology.orchestra.run.smithi164.stdout: key = AQAWSuVmQqEvDhAAHf4QjU2RjJVS5hfgppC5fg== 2024-09-14T08:33:39.844 INFO:teuthology.orchestra.run.smithi164.stdout: caps mon = "profile rbd-mirror" 2024-09-14T08:33:39.844 INFO:teuthology.orchestra.run.smithi164.stdout: caps osd = "profile rbd" 2024-09-14T08:33:39.844 INFO:teuthology.orchestra.run.smithi164.stderr:updated caps for client.mirror.3 2024-09-14T08:33:39.857 DEBUG:teuthology.orchestra.run.smithi164:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror mon '"'"'profile rbd-mirror-peer'"'"' osd '"'"'profile rbd'"'"'' 2024-09-14T08:33:40.219 INFO:teuthology.orchestra.run.smithi164.stdout:[client.mirror] 2024-09-14T08:33:40.219 INFO:teuthology.orchestra.run.smithi164.stdout: key = AQA8SuVmyDeDLBAAtB27M66Y1S7CzX1iEl8gLw== 2024-09-14T08:33:40.219 INFO:teuthology.orchestra.run.smithi164.stdout: caps mon = "profile rbd-mirror-peer" 2024-09-14T08:33:40.219 INFO:teuthology.orchestra.run.smithi164.stdout: caps osd = "profile rbd" 2024-09-14T08:33:40.220 INFO:teuthology.orchestra.run.smithi164.stderr:updated caps for client.mirror 2024-09-14T08:33:40.233 DEBUG:teuthology.orchestra.run.smithi164:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.0 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2024-09-14T08:33:40.597 INFO:teuthology.orchestra.run.smithi164.stdout:[client.mirror.0] 2024-09-14T08:33:40.597 INFO:teuthology.orchestra.run.smithi164.stdout: key = AQA8SuVmZ9EWMhAAXByDQRr+Wu+algMYlyxRVw== 2024-09-14T08:33:40.597 INFO:teuthology.orchestra.run.smithi164.stdout: caps mon = "profile rbd-mirror" 2024-09-14T08:33:40.597 INFO:teuthology.orchestra.run.smithi164.stdout: caps osd = "profile rbd" 2024-09-14T08:33:40.597 INFO:teuthology.orchestra.run.smithi164.stderr:updated caps for client.mirror.0 2024-09-14T08:33:40.611 DEBUG:teuthology.orchestra.run.smithi164:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.1 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2024-09-14T08:33:40.973 INFO:teuthology.orchestra.run.smithi164.stdout:[client.mirror.1] 2024-09-14T08:33:40.973 INFO:teuthology.orchestra.run.smithi164.stdout: key = AQA8SuVmL1upNxAAALAGO2uYGL1KLA/rcvDDFA== 2024-09-14T08:33:40.973 INFO:teuthology.orchestra.run.smithi164.stdout: caps mon = "profile rbd-mirror" 2024-09-14T08:33:40.973 INFO:teuthology.orchestra.run.smithi164.stdout: caps osd = "profile rbd" 2024-09-14T08:33:40.973 INFO:teuthology.orchestra.run.smithi164.stderr:updated caps for client.mirror.1 2024-09-14T08:33:40.987 DEBUG:teuthology.orchestra.run.smithi164:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.2 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2024-09-14T08:33:41.351 INFO:teuthology.orchestra.run.smithi164.stdout:[client.mirror.2] 2024-09-14T08:33:41.351 INFO:teuthology.orchestra.run.smithi164.stdout: key = AQA9SuVmU4yyARAA4bQupWUKxLRYUiw90CMmbg== 2024-09-14T08:33:41.351 INFO:teuthology.orchestra.run.smithi164.stdout: caps mon = "profile rbd-mirror" 2024-09-14T08:33:41.351 INFO:teuthology.orchestra.run.smithi164.stdout: caps osd = "profile rbd" 2024-09-14T08:33:41.351 INFO:teuthology.orchestra.run.smithi164.stderr:updated caps for client.mirror.2 2024-09-14T08:33:41.365 DEBUG:teuthology.orchestra.run.smithi164:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.3 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2024-09-14T08:33:41.729 INFO:teuthology.orchestra.run.smithi164.stdout:[client.mirror.3] 2024-09-14T08:33:41.729 INFO:teuthology.orchestra.run.smithi164.stdout: key = AQA9SuVmyZJMBxAAyN4nTCyrPgIYckUUYnpeog== 2024-09-14T08:33:41.729 INFO:teuthology.orchestra.run.smithi164.stdout: caps mon = "profile rbd-mirror" 2024-09-14T08:33:41.729 INFO:teuthology.orchestra.run.smithi164.stdout: caps osd = "profile rbd" 2024-09-14T08:33:41.729 INFO:teuthology.orchestra.run.smithi164.stderr:updated caps for client.mirror.3 2024-09-14T08:33:41.742 INFO:teuthology.run_tasks:Running task rbd-mirror... 2024-09-14T08:33:41.752 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Restarting daemon 2024-09-14T08:33:41.753 DEBUG:teuthology.orchestra.run.smithi164:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 2024-09-14T08:33:41.755 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Started 2024-09-14T08:33:41.755 INFO:teuthology.run_tasks:Running task rbd-mirror... 2024-09-14T08:33:41.769 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Restarting daemon 2024-09-14T08:33:41.769 DEBUG:teuthology.orchestra.run.smithi164:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 2024-09-14T08:33:41.771 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Started 2024-09-14T08:33:41.771 INFO:teuthology.run_tasks:Running task rbd-mirror... 2024-09-14T08:33:41.780 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Restarting daemon 2024-09-14T08:33:41.780 DEBUG:teuthology.orchestra.run.smithi164:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 2024-09-14T08:33:41.782 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Started 2024-09-14T08:33:41.782 INFO:teuthology.run_tasks:Running task rbd-mirror... 2024-09-14T08:33:41.792 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Restarting daemon 2024-09-14T08:33:41.792 DEBUG:teuthology.orchestra.run.smithi164:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.3 2024-09-14T08:33:41.794 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Started 2024-09-14T08:33:41.794 INFO:teuthology.run_tasks:Running task rbd-mirror... 2024-09-14T08:33:41.803 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Restarting daemon 2024-09-14T08:33:41.803 DEBUG:teuthology.orchestra.run.smithi164:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 2024-09-14T08:33:41.805 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Started 2024-09-14T08:33:41.805 INFO:teuthology.run_tasks:Running task rbd-mirror... 2024-09-14T08:33:41.815 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Restarting daemon 2024-09-14T08:33:41.815 DEBUG:teuthology.orchestra.run.smithi164:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 2024-09-14T08:33:41.817 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Started 2024-09-14T08:33:41.817 INFO:teuthology.run_tasks:Running task rbd-mirror... 2024-09-14T08:33:41.827 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Restarting daemon 2024-09-14T08:33:41.827 DEBUG:teuthology.orchestra.run.smithi164:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2 2024-09-14T08:33:41.829 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Started 2024-09-14T08:33:41.829 INFO:teuthology.run_tasks:Running task rbd-mirror... 2024-09-14T08:33:41.839 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Restarting daemon 2024-09-14T08:33:41.839 DEBUG:teuthology.orchestra.run.smithi164:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.3 2024-09-14T08:33:41.841 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Started 2024-09-14T08:33:41.842 INFO:teuthology.run_tasks:Running task rbd-mirror-thrash... 2024-09-14T08:33:41.852 INFO:tasks.rbd_mirror_thrash:rbd_mirror_thrash using random seed: 1726302821 2024-09-14T08:33:41.852 DEBUG:tasks.rbd_mirror_thrash:Yielding 2024-09-14T08:33:41.852 INFO:teuthology.run_tasks:Running task rbd-mirror-thrash... 2024-09-14T08:33:41.863 INFO:tasks.rbd_mirror_thrash:rbd_mirror_thrash using random seed: 1726302821 2024-09-14T08:33:41.864 DEBUG:tasks.rbd_mirror_thrash:Yielding 2024-09-14T08:33:41.864 INFO:teuthology.run_tasks:Running task workunit... 2024-09-14T08:33:41.878 INFO:tasks.workunit:Pulling workunits from ref 0d61375b8ab53be4c233f5b93e463edfdff9316a 2024-09-14T08:33:41.878 INFO:tasks.workunit:Making a separate scratch dir for every client... 2024-09-14T08:33:41.878 DEBUG:teuthology.orchestra.run.smithi164:> stat -- /home/ubuntu/cephtest/mnt.cluster1.mirror 2024-09-14T08:33:41.879 INFO:tasks.rbd_mirror_thrash:starting thrash for cluster cluster1 2024-09-14T08:33:41.879 INFO:tasks.rbd_mirror_thrash:waiting for 65 secs before thrashing 2024-09-14T08:33:41.879 INFO:tasks.rbd_mirror_thrash:starting thrash for cluster cluster2 2024-09-14T08:33:41.879 INFO:tasks.rbd_mirror_thrash:waiting for 65 secs before thrashing 2024-09-14T08:33:41.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T08:33:41.898 INFO:teuthology.orchestra.run.smithi164.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.cluster1.mirror': No such file or directory 2024-09-14T08:33:41.898 DEBUG:teuthology.orchestra.run.smithi164:> mkdir -- /home/ubuntu/cephtest/mnt.cluster1.mirror 2024-09-14T08:33:41.954 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.cluster1.mirror 2024-09-14T08:33:41.954 DEBUG:teuthology.orchestra.run.smithi164:> cd -- /home/ubuntu/cephtest/mnt.cluster1.mirror && mkdir -- client.mirror 2024-09-14T08:33:42.009 INFO:tasks.workunit:timeout=6h 2024-09-14T08:33:42.009 INFO:tasks.workunit:cleanup=True 2024-09-14T08:33:42.010 DEBUG:teuthology.orchestra.run.smithi164:> rm -rf /home/ubuntu/cephtest/clone.cluster1.client.mirror && git clone https://git.ceph.com/ceph.git /home/ubuntu/cephtest/clone.cluster1.client.mirror && cd /home/ubuntu/cephtest/clone.cluster1.client.mirror && git checkout 0d61375b8ab53be4c233f5b93e463edfdff9316a 2024-09-14T08:33:42.066 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Cloning into '/home/ubuntu/cephtest/clone.cluster1.client.mirror'... 2024-09-14T08:34:38.050 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Note: switching to '0d61375b8ab53be4c233f5b93e463edfdff9316a'. 2024-09-14T08:34:38.051 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:34:38.051 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:You are in 'detached HEAD' state. You can look around, make experimental 2024-09-14T08:34:38.051 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:changes and commit them, and you can discard any commits you make in this 2024-09-14T08:34:38.051 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:state without impacting any branches by switching back to a branch. 2024-09-14T08:34:38.051 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:34:38.051 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:If you want to create a new branch to retain commits you create, you may 2024-09-14T08:34:38.051 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:do so (now or later) by using -c with the switch command. Example: 2024-09-14T08:34:38.051 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:34:38.051 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: git switch -c 2024-09-14T08:34:38.051 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:34:38.052 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Or undo this operation with: 2024-09-14T08:34:38.052 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:34:38.052 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: git switch - 2024-09-14T08:34:38.052 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:34:38.052 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Turn off this advice by setting config variable advice.detachedHead to false 2024-09-14T08:34:38.052 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:34:38.052 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:HEAD is now at 0d61375b8ab Merge pull request #59485 from mcv21/cephadm_image_warn 2024-09-14T08:34:38.053 DEBUG:teuthology.orchestra.run.smithi164:> cd -- /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.cluster1.client.mirror 2024-09-14T08:34:38.154 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2024-09-14T08:34:38.156 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/direct_io' 2024-09-14T08:34:38.156 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2024-09-14T08:34:38.891 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2024-09-14T08:34:38.973 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2024-09-14T08:34:39.015 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/direct_io' 2024-09-14T08:34:39.017 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/fs' 2024-09-14T08:34:39.017 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2024-09-14T08:34:39.062 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/fs' 2024-09-14T08:34:39.062 DEBUG:teuthology.orchestra.run.smithi164:> set -ex 2024-09-14T08:34:39.062 DEBUG:teuthology.orchestra.run.smithi164:> dd if=/home/ubuntu/cephtest/workunits.list.cluster1.client.mirror of=/dev/stdout 2024-09-14T08:34:39.078 INFO:tasks.workunit:Running workunits matching rbd/rbd_mirror_stress.sh on cluster1.client.mirror... 2024-09-14T08:34:39.080 INFO:tasks.workunit:Running workunit rbd/rbd_mirror_stress.sh... 2024-09-14T08:34:39.080 DEBUG:teuthology.orchestra.run.smithi164:workunit test rbd/rbd_mirror_stress.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror/tmp && cd -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=0d61375b8ab53be4c233f5b93e463edfdff9316a TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster cluster1" CEPH_ID="mirror" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.cluster1.client.mirror CEPH_ROOT=/home/ubuntu/cephtest/clone.cluster1.client.mirror CEPH_MNT=/home/ubuntu/cephtest/mnt.cluster1.mirror CEPH_ARGS='' MIRROR_POOL_MODE=image RBD_IMAGE_FEATURES=layering RBD_MIRROR_INSTANCES=4 RBD_MIRROR_MODE=snapshot RBD_MIRROR_USE_EXISTING_CLUSTER=1 RBD_MIRROR_USE_RBD_MIRROR=1 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 6h /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/rbd_mirror_stress.sh 2024-09-14T08:34:39.138 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ IMAGE_COUNT=50 2024-09-14T08:34:39.138 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ export LOCKDEP=0 2024-09-14T08:34:39.138 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ LOCKDEP=0 2024-09-14T08:34:39.138 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ dirname /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/rbd_mirror_stress.sh 2024-09-14T08:34:39.140 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ . /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/rbd_mirror_helpers.sh 2024-09-14T08:34:39.140 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ RBD_MIRROR_INSTANCES=4 2024-09-14T08:34:39.140 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ CLUSTER1=cluster1 2024-09-14T08:34:39.140 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ CLUSTER2=cluster2 2024-09-14T08:34:39.140 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ PEER_CLUSTER_SUFFIX= 2024-09-14T08:34:39.140 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ POOL=mirror 2024-09-14T08:34:39.140 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ PARENT_POOL=mirror_parent 2024-09-14T08:34:39.140 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ NS1=ns1 2024-09-14T08:34:39.140 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ NS2=ns2 2024-09-14T08:34:39.140 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ TEMPDIR= 2024-09-14T08:34:39.140 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ CEPH_ID=mirror 2024-09-14T08:34:39.141 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ RBD_IMAGE_FEATURES=layering 2024-09-14T08:34:39.141 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ MIRROR_USER_ID_PREFIX=mirror. 2024-09-14T08:34:39.141 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ RBD_MIRROR_MODE=snapshot 2024-09-14T08:34:39.141 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ MIRROR_POOL_MODE=image 2024-09-14T08:34:39.141 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' snapshot = snapshot ']' 2024-09-14T08:34:39.141 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ MIRROR_POOL_MODE=image 2024-09-14T08:34:39.141 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ export 'CEPH_ARGS=--id mirror' 2024-09-14T08:34:39.141 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ CEPH_ARGS='--id mirror' 2024-09-14T08:34:39.141 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ LAST_MIRROR_INSTANCE=3 2024-09-14T08:34:39.141 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ dirname /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/rbd_mirror_stress.sh 2024-09-14T08:34:39.142 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ readlink -f /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/../../../src 2024-09-14T08:34:39.142 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ CEPH_ROOT=/home/ubuntu/cephtest/clone.cluster1.client.mirror/src 2024-09-14T08:34:39.142 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ CEPH_BIN=. 2024-09-14T08:34:39.142 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ CEPH_SRC=. 2024-09-14T08:34:39.142 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' -e CMakeCache.txt ']' 2024-09-14T08:34:39.143 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 0 -gt 0 ']' 2024-09-14T08:34:39.143 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ setup 2024-09-14T08:34:39.143 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local c 2024-09-14T08:34:39.143 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ trap 'cleanup $?' INT TERM EXIT 2024-09-14T08:34:39.144 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ setup_tempdir 2024-09-14T08:34:39.144 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n '' ']' 2024-09-14T08:34:39.144 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mktemp -d 2024-09-14T08:34:39.145 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ TEMPDIR=/tmp/tmp.IUd9LESfOA 2024-09-14T08:34:39.145 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -z 1 ']' 2024-09-14T08:34:39.145 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ setup_pools cluster1 cluster2 2024-09-14T08:34:39.145 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:34:39.145 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local remote_cluster=cluster2 2024-09-14T08:34:39.145 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mon_map_file 2024-09-14T08:34:39.145 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mon_addr 2024-09-14T08:34:39.145 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local admin_key_file 2024-09-14T08:34:39.145 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local uuid 2024-09-14T08:34:39.145 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ CEPH_ARGS= 2024-09-14T08:34:39.145 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ ceph --cluster cluster1 osd pool create mirror 64 64 2024-09-14T08:34:40.497 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:pool 'mirror' already exists 2024-09-14T08:34:40.508 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ CEPH_ARGS= 2024-09-14T08:34:40.508 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ ceph --cluster cluster1 osd pool create mirror_parent 64 64 2024-09-14T08:34:41.497 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:pool 'mirror_parent' already exists 2024-09-14T08:34:41.508 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ CEPH_ARGS= 2024-09-14T08:34:41.508 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 pool init mirror 2024-09-14T08:34:44.423 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ CEPH_ARGS= 2024-09-14T08:34:44.423 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 pool init mirror_parent 2024-09-14T08:34:46.880 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.26905428054195013) > weight (0.25) 2024-09-14T08:34:46.880 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.9745580426280032) > weight (0.25) 2024-09-14T08:34:46.880 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.9356585715799274) > weight (0.25) 2024-09-14T08:34:46.881 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.3 with skip (0.6176140107540483) > weight (0.25) 2024-09-14T08:34:46.881 INFO:tasks.rbd_mirror_thrash:waiting for 62 secs before thrashing 2024-09-14T08:34:46.881 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.0 2024-09-14T08:34:46.881 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Sent signal 15 2024-09-14T08:34:46.882 INFO:tasks.rbd_mirror_thrash:waiting for 8 secs before reviving daemons 2024-09-14T08:34:46.882 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi164.stderr:2024-09-14T08:34:46.883+0000 7f46c28d2640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 (PID: 36849) UID: 1000 2024-09-14T08:34:47.444 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n '' ']' 2024-09-14T08:34:47.445 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ CEPH_ARGS= 2024-09-14T08:34:47.445 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 mirror pool enable --site-name cluster1 mirror image 2024-09-14T08:34:47.498 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 mirror pool enable mirror_parent image 2024-09-14T08:34:47.542 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 namespace create mirror/ns1 2024-09-14T08:34:47.584 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 namespace create mirror/ns2 2024-09-14T08:34:47.623 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 mirror pool enable mirror/ns1 image 2024-09-14T08:34:47.665 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 mirror pool enable mirror/ns2 image 2024-09-14T08:34:47.708 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -z ']' 2024-09-14T08:34:47.708 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -z ']' 2024-09-14T08:34:47.708 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ peer_add cluster1 mirror cluster2 2024-09-14T08:34:47.708 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:34:47.708 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:34:47.708 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:34:47.708 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:34:47.708 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local client_cluster=cluster2 2024-09-14T08:34:47.708 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:34:47.708 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local remote_cluster=cluster2 2024-09-14T08:34:47.709 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local uuid_var_name 2024-09-14T08:34:47.709 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n '' ']' 2024-09-14T08:34:47.709 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local error_code 2024-09-14T08:34:47.709 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local peer_uuid 2024-09-14T08:34:47.709 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 16 32 2024-09-14T08:34:47.709 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set +e 2024-09-14T08:34:47.709 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster cluster1 mirror pool peer add mirror cluster2 2024-09-14T08:34:47.748 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ peer_uuid=5686d22d-6d25-4157-906e-b58cb392ec9e 2024-09-14T08:34:47.749 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ error_code=0 2024-09-14T08:34:47.749 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set -e 2024-09-14T08:34:47.749 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' 0 -eq 17 ']' 2024-09-14T08:34:47.749 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 0 -eq 0 2024-09-14T08:34:47.749 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n '' ']' 2024-09-14T08:34:47.749 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:34:47.749 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ peer_add cluster1 mirror_parent cluster2 2024-09-14T08:34:47.749 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:34:47.749 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:34:47.749 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror_parent 2024-09-14T08:34:47.749 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:34:47.749 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local client_cluster=cluster2 2024-09-14T08:34:47.749 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:34:47.750 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local remote_cluster=cluster2 2024-09-14T08:34:47.750 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local uuid_var_name 2024-09-14T08:34:47.750 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n '' ']' 2024-09-14T08:34:47.750 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local error_code 2024-09-14T08:34:47.750 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local peer_uuid 2024-09-14T08:34:47.750 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 16 32 2024-09-14T08:34:47.750 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set +e 2024-09-14T08:34:47.750 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster cluster1 mirror pool peer add mirror_parent cluster2 2024-09-14T08:34:47.789 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ peer_uuid=df1cc742-74eb-4c3c-8860-a969206b4587 2024-09-14T08:34:47.789 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ error_code=0 2024-09-14T08:34:47.789 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set -e 2024-09-14T08:34:47.790 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' 0 -eq 17 ']' 2024-09-14T08:34:47.790 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 0 -eq 0 2024-09-14T08:34:47.790 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n '' ']' 2024-09-14T08:34:47.790 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:34:47.790 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ setup_pools cluster2 cluster1 2024-09-14T08:34:47.790 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:34:47.790 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local remote_cluster=cluster1 2024-09-14T08:34:47.790 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mon_map_file 2024-09-14T08:34:47.790 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mon_addr 2024-09-14T08:34:47.790 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local admin_key_file 2024-09-14T08:34:47.790 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local uuid 2024-09-14T08:34:47.790 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ CEPH_ARGS= 2024-09-14T08:34:47.790 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ ceph --cluster cluster2 osd pool create mirror 64 64 2024-09-14T08:34:49.040 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:pool 'mirror' already exists 2024-09-14T08:34:49.052 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ CEPH_ARGS= 2024-09-14T08:34:49.052 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ ceph --cluster cluster2 osd pool create mirror_parent 64 64 2024-09-14T08:34:50.061 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:pool 'mirror_parent' already exists 2024-09-14T08:34:50.072 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ CEPH_ARGS= 2024-09-14T08:34:50.072 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 pool init mirror 2024-09-14T08:34:52.957 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ CEPH_ARGS= 2024-09-14T08:34:52.957 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 pool init mirror_parent 2024-09-14T08:34:54.883 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.0 2024-09-14T08:34:54.883 INFO:teuthology.orchestra.run:waiting for 600 2024-09-14T08:34:54.884 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.0 2024-09-14T08:34:54.884 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Restarting daemon 2024-09-14T08:34:54.884 DEBUG:teuthology.orchestra.run.smithi164:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 2024-09-14T08:34:54.887 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Started 2024-09-14T08:34:54.887 INFO:tasks.rbd_mirror_thrash:waiting for 98 secs before thrashing 2024-09-14T08:34:55.968 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n '' ']' 2024-09-14T08:34:55.968 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ CEPH_ARGS= 2024-09-14T08:34:55.968 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 mirror pool enable --site-name cluster2 mirror image 2024-09-14T08:34:56.021 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 mirror pool enable mirror_parent image 2024-09-14T08:34:56.063 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 namespace create mirror/ns1 2024-09-14T08:34:56.102 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 namespace create mirror/ns2 2024-09-14T08:34:56.144 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 mirror pool enable mirror/ns1 image 2024-09-14T08:34:56.184 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 mirror pool enable mirror/ns2 image 2024-09-14T08:34:56.228 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -z ']' 2024-09-14T08:34:56.228 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -z ']' 2024-09-14T08:34:56.228 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ peer_add cluster2 mirror cluster1 2024-09-14T08:34:56.228 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:34:56.228 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:34:56.228 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:34:56.229 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:34:56.229 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local client_cluster=cluster1 2024-09-14T08:34:56.229 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:34:56.229 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local remote_cluster=cluster1 2024-09-14T08:34:56.229 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local uuid_var_name 2024-09-14T08:34:56.229 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n '' ']' 2024-09-14T08:34:56.229 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local error_code 2024-09-14T08:34:56.229 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local peer_uuid 2024-09-14T08:34:56.229 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 16 32 2024-09-14T08:34:56.230 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set +e 2024-09-14T08:34:56.230 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster cluster2 mirror pool peer add mirror cluster1 2024-09-14T08:34:56.269 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ peer_uuid=660a1f8d-0b94-4cda-bc00-8780187ef4f9 2024-09-14T08:34:56.269 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ error_code=0 2024-09-14T08:34:56.269 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set -e 2024-09-14T08:34:56.269 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' 0 -eq 17 ']' 2024-09-14T08:34:56.269 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 0 -eq 0 2024-09-14T08:34:56.269 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n '' ']' 2024-09-14T08:34:56.269 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:34:56.269 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ peer_add cluster2 mirror_parent cluster1 2024-09-14T08:34:56.269 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:34:56.269 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:34:56.269 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror_parent 2024-09-14T08:34:56.269 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:34:56.270 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local client_cluster=cluster1 2024-09-14T08:34:56.270 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:34:56.270 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local remote_cluster=cluster1 2024-09-14T08:34:56.270 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local uuid_var_name 2024-09-14T08:34:56.270 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n '' ']' 2024-09-14T08:34:56.270 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local error_code 2024-09-14T08:34:56.270 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local peer_uuid 2024-09-14T08:34:56.270 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 16 32 2024-09-14T08:34:56.270 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set +e 2024-09-14T08:34:56.270 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster cluster2 mirror pool peer add mirror_parent cluster1 2024-09-14T08:34:56.307 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ peer_uuid=9de94a9a-7b67-4895-937f-f3ab8eaeef03 2024-09-14T08:34:56.308 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ error_code=0 2024-09-14T08:34:56.308 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set -e 2024-09-14T08:34:56.308 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' 0 -eq 17 ']' 2024-09-14T08:34:56.308 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 0 -eq 0 2024-09-14T08:34:56.308 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n '' ']' 2024-09-14T08:34:56.308 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:34:56.308 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n '' ']' 2024-09-14T08:34:56.308 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ start_mirrors cluster1 2024-09-14T08:34:56.308 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:34:56.308 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ seq 0 3 2024-09-14T08:34:56.327 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:34:56.327 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ start_mirror cluster1:0 2024-09-14T08:34:56.327 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1:0 2024-09-14T08:34:56.327 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local instance 2024-09-14T08:34:56.328 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T08:34:56.328 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local val=cluster1:0 2024-09-14T08:34:56.328 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster_var_name=cluster 2024-09-14T08:34:56.328 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local instance_var_name=instance 2024-09-14T08:34:56.328 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cluster=cluster1 2024-09-14T08:34:56.328 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ instance=0 2024-09-14T08:34:56.328 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T08:34:56.328 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ eval cluster=cluster1 2024-09-14T08:34:56.328 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:34:56.328 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ eval instance=0 2024-09-14T08:34:56.328 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=0 2024-09-14T08:34:56.328 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n 1 2024-09-14T08:34:56.328 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 2024-09-14T08:34:56.329 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:34:56.329 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ start_mirror cluster1:1 2024-09-14T08:34:56.329 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1:1 2024-09-14T08:34:56.329 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local instance 2024-09-14T08:34:56.329 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T08:34:56.329 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local val=cluster1:1 2024-09-14T08:34:56.330 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster_var_name=cluster 2024-09-14T08:34:56.330 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local instance_var_name=instance 2024-09-14T08:34:56.330 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cluster=cluster1 2024-09-14T08:34:56.330 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ instance=1 2024-09-14T08:34:56.330 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T08:34:56.330 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ eval cluster=cluster1 2024-09-14T08:34:56.331 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:34:56.331 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ eval instance=1 2024-09-14T08:34:56.331 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=1 2024-09-14T08:34:56.331 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n 1 2024-09-14T08:34:56.331 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 2024-09-14T08:34:56.332 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:34:56.332 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ start_mirror cluster1:2 2024-09-14T08:34:56.332 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1:2 2024-09-14T08:34:56.332 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local instance 2024-09-14T08:34:56.332 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T08:34:56.332 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local val=cluster1:2 2024-09-14T08:34:56.332 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster_var_name=cluster 2024-09-14T08:34:56.332 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local instance_var_name=instance 2024-09-14T08:34:56.332 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cluster=cluster1 2024-09-14T08:34:56.332 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ instance=2 2024-09-14T08:34:56.332 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T08:34:56.333 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ eval cluster=cluster1 2024-09-14T08:34:56.333 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:34:56.333 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ eval instance=2 2024-09-14T08:34:56.333 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=2 2024-09-14T08:34:56.333 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n 1 2024-09-14T08:34:56.333 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 2024-09-14T08:34:56.333 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:34:56.333 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ start_mirror cluster1:3 2024-09-14T08:34:56.333 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1:3 2024-09-14T08:34:56.333 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local instance 2024-09-14T08:34:56.333 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set_cluster_instance cluster1:3 cluster instance 2024-09-14T08:34:56.333 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local val=cluster1:3 2024-09-14T08:34:56.333 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster_var_name=cluster 2024-09-14T08:34:56.334 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local instance_var_name=instance 2024-09-14T08:34:56.334 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cluster=cluster1 2024-09-14T08:34:56.334 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ instance=3 2024-09-14T08:34:56.334 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' 3 = cluster1:3 ']' 2024-09-14T08:34:56.334 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ eval cluster=cluster1 2024-09-14T08:34:56.334 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:34:56.334 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ eval instance=3 2024-09-14T08:34:56.334 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=3 2024-09-14T08:34:56.334 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n 1 2024-09-14T08:34:56.334 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 2024-09-14T08:34:56.334 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ start_mirrors cluster2 2024-09-14T08:34:56.334 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:34:56.335 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ seq 0 3 2024-09-14T08:34:56.335 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:34:56.335 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ start_mirror cluster2:0 2024-09-14T08:34:56.335 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2:0 2024-09-14T08:34:56.335 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local instance 2024-09-14T08:34:56.335 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set_cluster_instance cluster2:0 cluster instance 2024-09-14T08:34:56.335 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local val=cluster2:0 2024-09-14T08:34:56.335 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster_var_name=cluster 2024-09-14T08:34:56.335 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local instance_var_name=instance 2024-09-14T08:34:56.336 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cluster=cluster2 2024-09-14T08:34:56.336 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ instance=0 2024-09-14T08:34:56.336 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' 0 = cluster2:0 ']' 2024-09-14T08:34:56.336 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ eval cluster=cluster2 2024-09-14T08:34:56.336 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster2 2024-09-14T08:34:56.336 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ eval instance=0 2024-09-14T08:34:56.336 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=0 2024-09-14T08:34:56.336 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n 1 2024-09-14T08:34:56.336 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 2024-09-14T08:34:56.336 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:34:56.336 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ start_mirror cluster2:1 2024-09-14T08:34:56.336 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2:1 2024-09-14T08:34:56.337 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local instance 2024-09-14T08:34:56.337 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set_cluster_instance cluster2:1 cluster instance 2024-09-14T08:34:56.337 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local val=cluster2:1 2024-09-14T08:34:56.337 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster_var_name=cluster 2024-09-14T08:34:56.337 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local instance_var_name=instance 2024-09-14T08:34:56.337 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cluster=cluster2 2024-09-14T08:34:56.337 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ instance=1 2024-09-14T08:34:56.337 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' 1 = cluster2:1 ']' 2024-09-14T08:34:56.337 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ eval cluster=cluster2 2024-09-14T08:34:56.337 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster2 2024-09-14T08:34:56.338 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ eval instance=1 2024-09-14T08:34:56.338 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=1 2024-09-14T08:34:56.338 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n 1 2024-09-14T08:34:56.338 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 2024-09-14T08:34:56.338 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:34:56.338 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ start_mirror cluster2:2 2024-09-14T08:34:56.338 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2:2 2024-09-14T08:34:56.338 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local instance 2024-09-14T08:34:56.338 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set_cluster_instance cluster2:2 cluster instance 2024-09-14T08:34:56.338 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local val=cluster2:2 2024-09-14T08:34:56.339 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster_var_name=cluster 2024-09-14T08:34:56.339 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local instance_var_name=instance 2024-09-14T08:34:56.339 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cluster=cluster2 2024-09-14T08:34:56.339 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ instance=2 2024-09-14T08:34:56.339 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' 2 = cluster2:2 ']' 2024-09-14T08:34:56.339 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ eval cluster=cluster2 2024-09-14T08:34:56.339 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster2 2024-09-14T08:34:56.339 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ eval instance=2 2024-09-14T08:34:56.339 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=2 2024-09-14T08:34:56.339 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n 1 2024-09-14T08:34:56.339 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 2024-09-14T08:34:56.339 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:34:56.339 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ start_mirror cluster2:3 2024-09-14T08:34:56.339 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2:3 2024-09-14T08:34:56.339 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local instance 2024-09-14T08:34:56.340 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set_cluster_instance cluster2:3 cluster instance 2024-09-14T08:34:56.340 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local val=cluster2:3 2024-09-14T08:34:56.340 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster_var_name=cluster 2024-09-14T08:34:56.340 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local instance_var_name=instance 2024-09-14T08:34:56.340 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cluster=cluster2 2024-09-14T08:34:56.340 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ instance=3 2024-09-14T08:34:56.340 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' 3 = cluster2:3 ']' 2024-09-14T08:34:56.340 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ eval cluster=cluster2 2024-09-14T08:34:56.340 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster2 2024-09-14T08:34:56.340 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ eval instance=3 2024-09-14T08:34:56.340 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=3 2024-09-14T08:34:56.340 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n 1 2024-09-14T08:34:56.341 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 2024-09-14T08:34:56.341 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ testlog 'TEST: add image and test replay after client crashes' 2024-09-14T08:34:56.341 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ tee -a /tmp/tmp.IUd9LESfOA/rbd-mirror.test.log 2024-09-14T08:34:56.341 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ date '+%F %T' 2024-09-14T08:34:56.341 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ echo 2024-09-14 08:34:56 TEST: add image and test replay after client crashes 2024-09-14T08:34:56.341 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:2024-09-14 08:34:56 TEST: add image and test replay after client crashes 2024-09-14T08:34:56.341 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=test 2024-09-14T08:34:56.341 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image_and_enable_mirror cluster2 mirror test snapshot 512M 2024-09-14T08:34:56.341 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:34:56.342 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:34:56.342 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:34:56.342 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:34:56.342 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:34:56.342 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:34:56.342 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:34:56.342 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n snapshot ']' 2024-09-14T08:34:56.342 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:34:56.342 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image cluster2 mirror test 512M 2024-09-14T08:34:56.342 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:34:56.342 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:34:56.342 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:34:56.342 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:34:56.342 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:34:56.343 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:34:56.343 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local size=128 2024-09-14T08:34:56.343 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n 512M ']' 2024-09-14T08:34:56.343 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ size=512M 2024-09-14T08:34:56.343 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:34:56.343 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering mirror/test 2024-09-14T08:34:56.370 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T08:34:56.370 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:34:56.370 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' image = image ']' 2024-09-14T08:34:56.370 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:34:56.370 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ enable_mirror cluster2 mirror test snapshot 2024-09-14T08:34:56.370 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:34:56.370 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:34:56.370 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:34:56.370 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:34:56.370 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/test snapshot 2024-09-14T08:34:57.031 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Mirroring enabled 2024-09-14T08:34:57.035 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 info mirror/test 2024-09-14T08:34:57.075 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'test': 2024-09-14T08:34:57.075 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 512 MiB in 128 objects 2024-09-14T08:34:57.076 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:34:57.076 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 1 2024-09-14T08:34:57.076 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 10a816744b38 2024-09-14T08:34:57.076 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.10a816744b38 2024-09-14T08:34:57.076 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:34:57.076 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering 2024-09-14T08:34:57.076 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:34:57.076 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:34:57.076 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:34:56 2024 2024-09-14T08:34:57.076 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:34:56 2024 2024-09-14T08:34:57.076 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:34:56 2024 2024-09-14T08:34:57.076 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:34:57.076 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:34:57.077 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: 1159fdda-fb30-49ff-ba1c-60890cdd16a0 2024-09-14T08:34:57.077 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: true 2024-09-14T08:34:57.079 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_started cluster1 mirror test 2024-09-14T08:34:57.079 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:34:57.079 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:34:57.079 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:34:57.079 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2024-09-14T08:34:57.079 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:34:57.080 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:34:57.080 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:34:57.080 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=started 2024-09-14T08:34:57.080 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:34:57.080 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:34:57.080 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:34:58.080 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_replay_state cluster1 mirror test started 2024-09-14T08:34:58.080 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:34:58.081 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:34:58.081 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:34:58.081 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=started 2024-09-14T08:34:58.081 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_result 2024-09-14T08:34:58.081 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=stopped 2024-09-14T08:34:58.082 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2024-09-14T08:34:58.082 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_instance=cluster1 2024-09-14T08:34:58.082 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:34:58.082 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ grep -i state 2024-09-14T08:34:58.082 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T08:34:58.082 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance=cluster1 2024-09-14T08:34:58.082 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local loop_instance 2024-09-14T08:34:58.082 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:34:58.082 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sleep 0 2024-09-14T08:34:58.083 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T08:34:58.083 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ seq 0 3 2024-09-14T08:34:58.084 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:34:58.085 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2024-09-14T08:34:58.085 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:0 2024-09-14T08:34:58.085 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:34:58.085 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:34:58.085 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T08:34:58.085 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:0 2024-09-14T08:34:58.085 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:34:58.085 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:34:58.085 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:34:58.085 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=0 2024-09-14T08:34:58.085 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T08:34:58.085 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:34:58.085 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:34:58.086 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=0 2024-09-14T08:34:58.086 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:34:58.086 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T08:34:58.086 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:0 2024-09-14T08:34:58.086 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:34:58.086 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:34:58.086 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T08:34:58.087 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:0 2024-09-14T08:34:58.087 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:34:58.087 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:34:58.087 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:34:58.088 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:34:58.088 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T08:34:58.088 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:34:58.088 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:34:58.088 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=0 2024-09-14T08:34:58.088 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=0 2024-09-14T08:34:58.088 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T08:34:58.106 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:34:58.106 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:34:58.107 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:34:58.107 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2024-09-14T08:34:58.182 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T08:34:58.182 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:34:58.182 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:34:58.185 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:34:58.185 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2024-09-14T08:34:58.185 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:1 2024-09-14T08:34:58.185 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:34:58.185 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:34:58.185 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T08:34:58.185 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:1 2024-09-14T08:34:58.186 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:34:58.186 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:34:58.186 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:34:58.186 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=1 2024-09-14T08:34:58.186 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T08:34:58.186 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:34:58.186 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:34:58.186 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=1 2024-09-14T08:34:58.186 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:34:58.186 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T08:34:58.186 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:1 2024-09-14T08:34:58.186 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:34:58.186 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:34:58.186 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T08:34:58.186 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:1 2024-09-14T08:34:58.187 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:34:58.187 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:34:58.187 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:34:58.187 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:34:58.187 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T08:34:58.187 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:34:58.187 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:34:58.187 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=1 2024-09-14T08:34:58.187 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=1 2024-09-14T08:34:58.187 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T08:34:58.203 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:34:58.204 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:34:58.204 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:34:58.204 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2024-09-14T08:34:58.274 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T08:34:58.274 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:34:58.274 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:34:58.277 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:34:58.277 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2024-09-14T08:34:58.277 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:2 2024-09-14T08:34:58.277 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:34:58.277 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:34:58.277 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-14T08:34:58.277 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:2 2024-09-14T08:34:58.277 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:34:58.277 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:34:58.277 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:34:58.277 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=2 2024-09-14T08:34:58.277 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T08:34:58.278 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:34:58.278 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:34:58.278 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=2 2024-09-14T08:34:58.278 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:34:58.278 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-14T08:34:58.278 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:2 2024-09-14T08:34:58.278 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:34:58.278 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:34:58.278 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T08:34:58.278 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:2 2024-09-14T08:34:58.278 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:34:58.278 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:34:58.278 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:34:58.278 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:34:58.279 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-14T08:34:58.279 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:34:58.279 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:34:58.279 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=2 2024-09-14T08:34:58.279 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=2 2024-09-14T08:34:58.279 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T08:34:58.296 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:34:58.296 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:34:58.296 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:34:58.297 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2024-09-14T08:34:58.365 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T08:34:58.365 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:34:58.365 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:34:58.368 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:34:58.368 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2024-09-14T08:34:58.368 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:3 2024-09-14T08:34:58.368 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:34:58.368 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:34:58.368 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:3 cluster instance 2024-09-14T08:34:58.368 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:3 2024-09-14T08:34:58.368 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:34:58.368 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:34:58.369 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:34:58.369 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=3 2024-09-14T08:34:58.369 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-14T08:34:58.369 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:34:58.369 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:34:58.369 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=3 2024-09-14T08:34:58.369 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=3 2024-09-14T08:34:58.369 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:3 cluster1 2024-09-14T08:34:58.369 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:3 2024-09-14T08:34:58.369 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:34:58.369 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:34:58.369 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-14T08:34:58.369 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:3 2024-09-14T08:34:58.370 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:34:58.370 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:34:58.370 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:34:58.370 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=3 2024-09-14T08:34:58.370 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 3 = cluster1:3 ']' 2024-09-14T08:34:58.370 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:34:58.370 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:34:58.370 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=3 2024-09-14T08:34:58.370 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=3 2024-09-14T08:34:58.370 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-14T08:34:58.387 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T08:34:58.387 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T08:34:58.387 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T08:34:58.387 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2024-09-14T08:34:58.458 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T08:34:58.458 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:34:58.458 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:34:58.459 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:34:58.459 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sleep 1 2024-09-14T08:34:59.461 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T08:34:59.461 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ seq 0 3 2024-09-14T08:34:59.464 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:34:59.464 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2024-09-14T08:34:59.464 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:0 2024-09-14T08:34:59.464 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:34:59.464 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:34:59.464 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T08:34:59.464 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:0 2024-09-14T08:34:59.464 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:34:59.464 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:34:59.464 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:34:59.464 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=0 2024-09-14T08:34:59.465 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T08:34:59.465 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:34:59.465 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:34:59.465 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=0 2024-09-14T08:34:59.465 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:34:59.465 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T08:34:59.465 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:0 2024-09-14T08:34:59.465 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:34:59.465 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:34:59.465 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T08:34:59.465 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:0 2024-09-14T08:34:59.465 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:34:59.465 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:34:59.466 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:34:59.466 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:34:59.466 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T08:34:59.466 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:34:59.466 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:34:59.466 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=0 2024-09-14T08:34:59.466 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=0 2024-09-14T08:34:59.466 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T08:34:59.484 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:34:59.484 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:34:59.485 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:34:59.485 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2024-09-14T08:34:59.559 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T08:34:59.559 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:34:59.559 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:34:59.562 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:34:59.562 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2024-09-14T08:34:59.562 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:1 2024-09-14T08:34:59.562 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:34:59.562 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:34:59.562 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T08:34:59.562 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:1 2024-09-14T08:34:59.562 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:34:59.562 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:34:59.563 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:34:59.563 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=1 2024-09-14T08:34:59.563 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T08:34:59.563 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:34:59.563 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:34:59.563 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=1 2024-09-14T08:34:59.563 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:34:59.563 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T08:34:59.563 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:1 2024-09-14T08:34:59.563 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:34:59.563 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:34:59.563 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T08:34:59.563 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:1 2024-09-14T08:34:59.563 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:34:59.564 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:34:59.564 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:34:59.564 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:34:59.564 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T08:34:59.564 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:34:59.564 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:34:59.564 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=1 2024-09-14T08:34:59.564 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=1 2024-09-14T08:34:59.564 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T08:34:59.580 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:34:59.581 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:34:59.581 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:34:59.581 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2024-09-14T08:34:59.650 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T08:34:59.650 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:34:59.650 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:34:59.653 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:34:59.653 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2024-09-14T08:34:59.653 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:2 2024-09-14T08:34:59.653 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:34:59.653 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:34:59.653 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-14T08:34:59.653 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:2 2024-09-14T08:34:59.653 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:34:59.654 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:34:59.654 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:34:59.654 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=2 2024-09-14T08:34:59.654 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T08:34:59.654 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:34:59.654 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:34:59.654 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=2 2024-09-14T08:34:59.654 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:34:59.654 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-14T08:34:59.654 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:2 2024-09-14T08:34:59.654 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:34:59.654 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:34:59.654 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T08:34:59.654 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:2 2024-09-14T08:34:59.655 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:34:59.655 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:34:59.655 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:34:59.655 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:34:59.655 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-14T08:34:59.655 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:34:59.655 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:34:59.655 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=2 2024-09-14T08:34:59.655 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=2 2024-09-14T08:34:59.655 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T08:34:59.672 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:34:59.672 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:34:59.673 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:34:59.673 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2024-09-14T08:34:59.743 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T08:34:59.743 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:34:59.743 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:34:59.746 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:34:59.746 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2024-09-14T08:34:59.746 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:3 2024-09-14T08:34:59.746 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:34:59.746 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:34:59.747 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:3 cluster instance 2024-09-14T08:34:59.747 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:3 2024-09-14T08:34:59.747 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:34:59.747 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:34:59.747 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:34:59.747 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=3 2024-09-14T08:34:59.747 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-14T08:34:59.747 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:34:59.747 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:34:59.747 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=3 2024-09-14T08:34:59.747 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=3 2024-09-14T08:34:59.747 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:3 cluster1 2024-09-14T08:34:59.747 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:3 2024-09-14T08:34:59.747 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:34:59.748 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:34:59.748 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-14T08:34:59.748 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:3 2024-09-14T08:34:59.748 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:34:59.748 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:34:59.748 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:34:59.748 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=3 2024-09-14T08:34:59.748 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 3 = cluster1:3 ']' 2024-09-14T08:34:59.748 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:34:59.748 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:34:59.748 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=3 2024-09-14T08:34:59.748 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=3 2024-09-14T08:34:59.748 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-14T08:34:59.764 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T08:34:59.764 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T08:34:59.765 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T08:34:59.765 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2024-09-14T08:34:59.834 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T08:34:59.834 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:34:59.834 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:34:59.835 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:34:59.835 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sleep 2 2024-09-14T08:35:01.836 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T08:35:01.837 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ seq 0 3 2024-09-14T08:35:01.839 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:35:01.839 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2024-09-14T08:35:01.839 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:0 2024-09-14T08:35:01.840 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:35:01.840 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:35:01.840 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T08:35:01.840 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:0 2024-09-14T08:35:01.840 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:35:01.840 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:35:01.840 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:35:01.840 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=0 2024-09-14T08:35:01.840 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T08:35:01.840 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:35:01.840 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:35:01.840 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=0 2024-09-14T08:35:01.840 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:35:01.841 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T08:35:01.841 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:0 2024-09-14T08:35:01.841 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:35:01.841 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:35:01.841 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T08:35:01.841 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:0 2024-09-14T08:35:01.841 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:35:01.841 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:35:01.841 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:35:01.841 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:35:01.841 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T08:35:01.841 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:35:01.841 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:35:01.841 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=0 2024-09-14T08:35:01.842 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=0 2024-09-14T08:35:01.842 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T08:35:01.861 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:35:01.861 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:35:01.862 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:35:01.862 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2024-09-14T08:35:01.939 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T08:35:01.939 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:35:01.939 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:35:01.942 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:35:01.942 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2024-09-14T08:35:01.942 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:1 2024-09-14T08:35:01.942 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:35:01.942 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:35:01.943 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T08:35:01.943 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:1 2024-09-14T08:35:01.943 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:35:01.943 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:35:01.943 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:35:01.943 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=1 2024-09-14T08:35:01.943 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T08:35:01.943 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:35:01.943 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:35:01.943 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=1 2024-09-14T08:35:01.943 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:35:01.943 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T08:35:01.943 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:1 2024-09-14T08:35:01.943 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:35:01.944 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:35:01.944 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T08:35:01.944 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:1 2024-09-14T08:35:01.944 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:35:01.944 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:35:01.944 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:35:01.944 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:35:01.944 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T08:35:01.944 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:35:01.944 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:35:01.944 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=1 2024-09-14T08:35:01.944 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=1 2024-09-14T08:35:01.944 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T08:35:01.961 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:35:01.961 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:35:01.961 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:35:01.961 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2024-09-14T08:35:02.032 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T08:35:02.032 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:35:02.032 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:35:02.035 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:35:02.035 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2024-09-14T08:35:02.035 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:2 2024-09-14T08:35:02.036 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:35:02.036 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:35:02.036 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-14T08:35:02.036 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:2 2024-09-14T08:35:02.036 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:35:02.036 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:35:02.036 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:35:02.036 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=2 2024-09-14T08:35:02.036 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T08:35:02.036 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:35:02.036 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:35:02.036 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=2 2024-09-14T08:35:02.036 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:35:02.036 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-14T08:35:02.037 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:2 2024-09-14T08:35:02.037 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:35:02.037 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:35:02.037 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T08:35:02.037 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:2 2024-09-14T08:35:02.037 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:35:02.037 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:35:02.037 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:35:02.037 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:35:02.037 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-14T08:35:02.037 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:35:02.037 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:35:02.037 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=2 2024-09-14T08:35:02.037 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=2 2024-09-14T08:35:02.038 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T08:35:02.055 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:35:02.055 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:35:02.055 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:35:02.055 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2024-09-14T08:35:02.125 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T08:35:02.125 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:35:02.126 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:35:02.129 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:35:02.129 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2024-09-14T08:35:02.129 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:3 2024-09-14T08:35:02.129 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:35:02.129 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:35:02.129 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:3 cluster instance 2024-09-14T08:35:02.129 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:3 2024-09-14T08:35:02.129 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:35:02.129 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:35:02.130 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:35:02.130 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=3 2024-09-14T08:35:02.130 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-14T08:35:02.130 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:35:02.130 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:35:02.130 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=3 2024-09-14T08:35:02.130 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=3 2024-09-14T08:35:02.130 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:3 cluster1 2024-09-14T08:35:02.130 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:3 2024-09-14T08:35:02.130 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:35:02.130 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:35:02.130 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-14T08:35:02.130 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:3 2024-09-14T08:35:02.130 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:35:02.130 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:35:02.131 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:35:02.131 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=3 2024-09-14T08:35:02.131 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 3 = cluster1:3 ']' 2024-09-14T08:35:02.131 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:35:02.131 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:35:02.131 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=3 2024-09-14T08:35:02.131 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=3 2024-09-14T08:35:02.131 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-14T08:35:02.147 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T08:35:02.147 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T08:35:02.147 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T08:35:02.147 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2024-09-14T08:35:02.216 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T08:35:02.216 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:35:02.216 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:35:02.217 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:35:02.217 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sleep 4 2024-09-14T08:35:06.219 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T08:35:06.219 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ seq 0 3 2024-09-14T08:35:06.222 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:35:06.222 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2024-09-14T08:35:06.222 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:0 2024-09-14T08:35:06.222 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:35:06.222 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:35:06.222 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T08:35:06.222 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:0 2024-09-14T08:35:06.222 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:35:06.222 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:35:06.223 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:35:06.223 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=0 2024-09-14T08:35:06.223 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T08:35:06.223 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:35:06.223 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:35:06.223 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=0 2024-09-14T08:35:06.223 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:35:06.223 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T08:35:06.223 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:0 2024-09-14T08:35:06.223 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:35:06.223 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:35:06.223 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T08:35:06.223 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:0 2024-09-14T08:35:06.223 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:35:06.224 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:35:06.224 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:35:06.224 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:35:06.224 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T08:35:06.224 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:35:06.224 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:35:06.224 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=0 2024-09-14T08:35:06.224 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=0 2024-09-14T08:35:06.224 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T08:35:06.245 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:35:06.245 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:35:06.245 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:35:06.245 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2024-09-14T08:35:06.320 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T08:35:06.320 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:35:06.321 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:35:06.324 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:35:06.324 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2024-09-14T08:35:06.324 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:1 2024-09-14T08:35:06.324 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:35:06.324 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:35:06.324 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T08:35:06.324 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:1 2024-09-14T08:35:06.324 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:35:06.324 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:35:06.324 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:35:06.324 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=1 2024-09-14T08:35:06.325 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T08:35:06.325 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:35:06.325 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:35:06.325 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=1 2024-09-14T08:35:06.325 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:35:06.325 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T08:35:06.325 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:1 2024-09-14T08:35:06.325 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:35:06.325 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:35:06.325 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T08:35:06.325 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:1 2024-09-14T08:35:06.325 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:35:06.325 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:35:06.325 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:35:06.325 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:35:06.326 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T08:35:06.326 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:35:06.326 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:35:06.326 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=1 2024-09-14T08:35:06.326 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=1 2024-09-14T08:35:06.326 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T08:35:06.342 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:35:06.342 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:35:06.342 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:35:06.343 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2024-09-14T08:35:06.414 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T08:35:06.414 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:35:06.414 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:35:06.417 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:35:06.417 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2024-09-14T08:35:06.417 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:2 2024-09-14T08:35:06.417 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:35:06.417 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:35:06.417 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-14T08:35:06.417 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:2 2024-09-14T08:35:06.417 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:35:06.417 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:35:06.417 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:35:06.417 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=2 2024-09-14T08:35:06.418 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T08:35:06.418 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:35:06.418 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:35:06.418 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=2 2024-09-14T08:35:06.418 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:35:06.418 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-14T08:35:06.418 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:2 2024-09-14T08:35:06.418 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:35:06.418 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:35:06.418 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T08:35:06.418 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:2 2024-09-14T08:35:06.418 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:35:06.418 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:35:06.418 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:35:06.419 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:35:06.419 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-14T08:35:06.419 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:35:06.419 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:35:06.419 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=2 2024-09-14T08:35:06.419 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=2 2024-09-14T08:35:06.419 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T08:35:06.435 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:35:06.435 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:35:06.435 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:35:06.435 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2024-09-14T08:35:06.506 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T08:35:06.506 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:35:06.506 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:35:06.509 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:35:06.509 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2024-09-14T08:35:06.509 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:3 2024-09-14T08:35:06.509 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:35:06.509 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:35:06.509 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:3 cluster instance 2024-09-14T08:35:06.509 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:3 2024-09-14T08:35:06.510 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:35:06.510 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:35:06.510 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:35:06.510 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=3 2024-09-14T08:35:06.510 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-14T08:35:06.510 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:35:06.510 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:35:06.510 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=3 2024-09-14T08:35:06.510 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=3 2024-09-14T08:35:06.510 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:3 cluster1 2024-09-14T08:35:06.510 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:3 2024-09-14T08:35:06.510 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:35:06.510 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:35:06.510 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-14T08:35:06.511 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:3 2024-09-14T08:35:06.511 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:35:06.511 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:35:06.511 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:35:06.511 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=3 2024-09-14T08:35:06.511 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 3 = cluster1:3 ']' 2024-09-14T08:35:06.511 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:35:06.511 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:35:06.511 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=3 2024-09-14T08:35:06.511 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=3 2024-09-14T08:35:06.511 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-14T08:35:06.529 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T08:35:06.529 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T08:35:06.529 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T08:35:06.529 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2024-09-14T08:35:06.599 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T08:35:06.599 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:35:06.599 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:35:06.600 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:35:06.600 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sleep 8 2024-09-14T08:35:14.601 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T08:35:14.602 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ seq 0 3 2024-09-14T08:35:14.604 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:35:14.605 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2024-09-14T08:35:14.605 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:0 2024-09-14T08:35:14.605 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:35:14.605 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:35:14.605 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T08:35:14.605 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:0 2024-09-14T08:35:14.605 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:35:14.605 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:35:14.605 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:35:14.605 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=0 2024-09-14T08:35:14.605 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T08:35:14.605 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:35:14.606 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:35:14.606 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=0 2024-09-14T08:35:14.606 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:35:14.606 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T08:35:14.606 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:0 2024-09-14T08:35:14.606 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:35:14.606 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:35:14.606 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T08:35:14.606 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:0 2024-09-14T08:35:14.606 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:35:14.606 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:35:14.606 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:35:14.606 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:35:14.606 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T08:35:14.606 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:35:14.607 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:35:14.607 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=0 2024-09-14T08:35:14.607 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=0 2024-09-14T08:35:14.607 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T08:35:14.626 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:35:14.626 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:35:14.626 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:35:14.626 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2024-09-14T08:35:14.712 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ return 0 2024-09-14T08:35:14.713 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T08:35:14.713 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ echo ' "state": "Replaying"' 2024-09-14T08:35:14.713 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep -i Replaying 2024-09-14T08:35:14.713 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: "state": "Replaying" 2024-09-14T08:35:14.714 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_state=started 2024-09-14T08:35:14.714 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test started = started 2024-09-14T08:35:14.714 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:35:14.714 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ clean_snap_name= 2024-09-14T08:35:14.714 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ seq 1 10 2024-09-14T08:35:14.715 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 10` 2024-09-14T08:35:14.715 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ stress_write_image cluster2 mirror test 2024-09-14T08:35:14.715 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:35:14.715 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:35:14.715 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:35:14.716 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T08:35:14.717 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local duration=9 2024-09-14T08:35:14.717 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set +e 2024-09-14T08:35:14.717 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ timeout 9s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2024-09-14T08:35:14.947 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:100 IOs, 68222976 bytes 2024-09-14T08:35:15.143 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:200 IOs, 126903296 bytes 2024-09-14T08:35:15.214 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:300 IOs, 133248000 bytes 2024-09-14T08:35:15.265 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:400 IOs, 133843968 bytes 2024-09-14T08:35:15.459 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:500 IOs, 200481792 bytes 2024-09-14T08:35:15.607 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:600 IOs, 247895040 bytes 2024-09-14T08:35:15.667 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:700 IOs, 262925312 bytes 2024-09-14T08:35:15.786 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:800 IOs, 299071488 bytes 2024-09-14T08:35:15.926 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:900 IOs, 334597120 bytes 2024-09-14T08:35:15.999 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1000 IOs, 345136128 bytes 2024-09-14T08:35:16.056 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1100 IOs, 354293760 bytes 2024-09-14T08:35:16.174 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1200 IOs, 388654080 bytes 2024-09-14T08:35:16.355 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1300 IOs, 463784960 bytes 2024-09-14T08:35:16.490 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1400 IOs, 497804288 bytes 2024-09-14T08:35:16.564 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1500 IOs, 516967424 bytes 2024-09-14T08:35:16.679 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1600 IOs, 544185344 bytes 2024-09-14T08:35:16.804 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1700 IOs, 584652800 bytes 2024-09-14T08:35:16.926 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1800 IOs, 620685312 bytes 2024-09-14T08:35:17.062 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1900 IOs, 661783552 bytes 2024-09-14T08:35:17.208 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2000 IOs, 701215744 bytes 2024-09-14T08:35:17.320 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2100 IOs, 725756928 bytes 2024-09-14T08:35:17.443 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2200 IOs, 755112960 bytes 2024-09-14T08:35:17.552 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2300 IOs, 780537856 bytes 2024-09-14T08:35:17.739 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2400 IOs, 842873856 bytes 2024-09-14T08:35:17.868 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2500 IOs, 880508928 bytes 2024-09-14T08:35:18.060 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2600 IOs, 933971968 bytes 2024-09-14T08:35:18.199 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2700 IOs, 969966592 bytes 2024-09-14T08:35:18.244 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2800 IOs, 970541056 bytes 2024-09-14T08:35:18.407 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2900 IOs, 1042743296 bytes 2024-09-14T08:35:18.620 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3000 IOs, 1118469120 bytes 2024-09-14T08:35:18.781 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3100 IOs, 1156062208 bytes 2024-09-14T08:35:18.896 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3200 IOs, 1176551424 bytes 2024-09-14T08:35:18.951 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3300 IOs, 1177177088 bytes 2024-09-14T08:35:19.139 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3400 IOs, 1235470336 bytes 2024-09-14T08:35:19.261 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3500 IOs, 1266615296 bytes 2024-09-14T08:35:19.472 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3600 IOs, 1341065216 bytes 2024-09-14T08:35:19.608 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3700 IOs, 1378532352 bytes 2024-09-14T08:35:19.788 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3800 IOs, 1421333504 bytes 2024-09-14T08:35:19.968 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3900 IOs, 1457316864 bytes 2024-09-14T08:35:20.225 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4000 IOs, 1497278464 bytes 2024-09-14T08:35:21.894 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4100 IOs, 1579195392 bytes 2024-09-14T08:35:22.025 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4200 IOs, 1601255424 bytes 2024-09-14T08:35:22.232 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4300 IOs, 1697950720 bytes 2024-09-14T08:35:22.575 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4400 IOs, 1719730176 bytes 2024-09-14T08:35:22.791 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4500 IOs, 1792109568 bytes 2024-09-14T08:35:23.154 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4600 IOs, 1909557248 bytes 2024-09-14T08:35:23.405 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4700 IOs, 1942962176 bytes 2024-09-14T08:35:23.556 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4800 IOs, 1970679808 bytes 2024-09-14T08:35:23.697 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4900 IOs, 2007774208 bytes 2024-09-14T08:35:23.725 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ error_code=124 2024-09-14T08:35:23.725 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set -e 2024-09-14T08:35:23.725 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T08:35:23.725 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:35:23.725 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2024-09-14T08:35:23.725 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:35:23.725 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:35:23.725 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:35:23.725 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state_pattern=up+replaying 2024-09-14T08:35:23.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local description_pattern= 2024-09-14T08:35:23.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local service_pattern= 2024-09-14T08:35:23.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:35:23.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:35:24.727 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2024-09-14T08:35:24.727 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:35:24.727 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:35:24.727 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:35:24.727 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state_pattern=up+replaying 2024-09-14T08:35:24.727 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local description_pattern= 2024-09-14T08:35:24.727 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local service_pattern= 2024-09-14T08:35:24.727 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status 2024-09-14T08:35:24.727 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ CEPH_ARGS= 2024-09-14T08:35:24.727 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2024-09-14T08:35:24.785 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ status='test: 2024-09-14T08:35:24.785 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: global_id: 1159fdda-fb30-49ff-ba1c-60890cdd16a0 2024-09-14T08:35:24.785 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: state: down+unknown 2024-09-14T08:35:24.785 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: description: status not found 2024-09-14T08:35:24.785 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: last_update: 2024-09-14T08:35:24.785 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: peer_sites: 2024-09-14T08:35:24.785 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: name: cluster2 2024-09-14T08:35:24.785 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: state: down+unknown 2024-09-14T08:35:24.786 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: description: status not found 2024-09-14T08:35:24.786 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: last_update: ' 2024-09-14T08:35:24.786 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep '^ state: .*up+replaying' 2024-09-14T08:35:24.786 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 1 2024-09-14T08:35:24.786 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:35:24.786 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 2 2024-09-14T08:35:26.788 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2024-09-14T08:35:26.788 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:35:26.788 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:35:26.788 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:35:26.788 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state_pattern=up+replaying 2024-09-14T08:35:26.788 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local description_pattern= 2024-09-14T08:35:26.788 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local service_pattern= 2024-09-14T08:35:26.788 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status 2024-09-14T08:35:26.788 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ CEPH_ARGS= 2024-09-14T08:35:26.789 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2024-09-14T08:35:26.844 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ status='test: 2024-09-14T08:35:26.845 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: global_id: 1159fdda-fb30-49ff-ba1c-60890cdd16a0 2024-09-14T08:35:26.845 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: state: down+unknown 2024-09-14T08:35:26.845 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: description: status not found 2024-09-14T08:35:26.845 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: last_update: 2024-09-14T08:35:26.845 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: peer_sites: 2024-09-14T08:35:26.845 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: name: cluster2 2024-09-14T08:35:26.845 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: state: down+unknown 2024-09-14T08:35:26.845 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: description: status not found 2024-09-14T08:35:26.845 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: last_update: ' 2024-09-14T08:35:26.845 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep '^ state: .*up+replaying' 2024-09-14T08:35:26.846 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 1 2024-09-14T08:35:26.846 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:35:26.846 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 4 2024-09-14T08:35:30.847 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2024-09-14T08:35:30.848 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:35:30.848 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:35:30.848 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:35:30.848 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state_pattern=up+replaying 2024-09-14T08:35:30.848 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local description_pattern= 2024-09-14T08:35:30.848 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local service_pattern= 2024-09-14T08:35:30.848 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status 2024-09-14T08:35:30.848 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ CEPH_ARGS= 2024-09-14T08:35:30.848 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2024-09-14T08:35:30.903 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ status='test: 2024-09-14T08:35:30.903 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: global_id: 1159fdda-fb30-49ff-ba1c-60890cdd16a0 2024-09-14T08:35:30.903 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: state: down+unknown 2024-09-14T08:35:30.903 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: description: status not found 2024-09-14T08:35:30.903 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: last_update: 2024-09-14T08:35:30.904 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: peer_sites: 2024-09-14T08:35:30.904 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: name: cluster2 2024-09-14T08:35:30.904 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: state: down+unknown 2024-09-14T08:35:30.904 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: description: status not found 2024-09-14T08:35:30.904 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: last_update: ' 2024-09-14T08:35:30.904 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep '^ state: .*up+replaying' 2024-09-14T08:35:30.904 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 1 2024-09-14T08:35:30.905 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:35:30.905 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 8 2024-09-14T08:35:38.906 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2024-09-14T08:35:38.907 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:35:38.907 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:35:38.907 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:35:38.907 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state_pattern=up+replaying 2024-09-14T08:35:38.907 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local description_pattern= 2024-09-14T08:35:38.907 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local service_pattern= 2024-09-14T08:35:38.907 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status 2024-09-14T08:35:38.907 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ CEPH_ARGS= 2024-09-14T08:35:38.907 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2024-09-14T08:35:38.961 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ status='test: 2024-09-14T08:35:38.961 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: global_id: 1159fdda-fb30-49ff-ba1c-60890cdd16a0 2024-09-14T08:35:38.961 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: state: down+unknown 2024-09-14T08:35:38.961 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: description: status not found 2024-09-14T08:35:38.961 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: last_update: 2024-09-14T08:35:38.961 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: peer_sites: 2024-09-14T08:35:38.961 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: name: cluster2 2024-09-14T08:35:38.962 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: state: down+unknown 2024-09-14T08:35:38.962 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: description: status not found 2024-09-14T08:35:38.962 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: last_update: ' 2024-09-14T08:35:38.962 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep '^ state: .*up+replaying' 2024-09-14T08:35:38.962 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 1 2024-09-14T08:35:38.962 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:35:38.963 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 8 2024-09-14T08:35:46.964 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2024-09-14T08:35:46.964 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:35:46.964 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:35:46.964 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:35:46.965 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state_pattern=up+replaying 2024-09-14T08:35:46.965 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local description_pattern= 2024-09-14T08:35:46.965 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local service_pattern= 2024-09-14T08:35:46.965 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status 2024-09-14T08:35:46.965 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ CEPH_ARGS= 2024-09-14T08:35:46.965 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2024-09-14T08:35:47.023 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ status='test: 2024-09-14T08:35:47.023 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: global_id: 1159fdda-fb30-49ff-ba1c-60890cdd16a0 2024-09-14T08:35:47.023 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: state: up+replaying 2024-09-14T08:35:47.023 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726302897,"remote_snapshot_timestamp":1726302897,"replay_state":"idle"} 2024-09-14T08:35:47.023 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: service: mirror.0 on smithi164 2024-09-14T08:35:47.023 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: last_update: 2024-09-14 08:35:42 2024-09-14T08:35:47.023 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: peer_sites: 2024-09-14T08:35:47.024 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: name: cluster2 2024-09-14T08:35:47.024 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: state: up+stopped 2024-09-14T08:35:47.024 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: description: local image is primary 2024-09-14T08:35:47.024 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: last_update: 2024-09-14 08:35:41' 2024-09-14T08:35:47.024 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep '^ state: .*up+replaying' 2024-09-14T08:35:47.024 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: state: up+replaying 2024-09-14T08:35:47.024 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep '^ description: .*' 2024-09-14T08:35:47.025 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726302897,"remote_snapshot_timestamp":1726302897,"replay_state":"idle"} 2024-09-14T08:35:47.026 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n '' ']' 2024-09-14T08:35:47.026 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ echo up+replaying 2024-09-14T08:35:47.026 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep '^up+' 2024-09-14T08:35:47.027 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:up+replaying 2024-09-14T08:35:47.027 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep 'service: *mirror..* on ' 2024-09-14T08:35:47.028 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: service: mirror.0 on smithi164 2024-09-14T08:35:47.028 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local last_update 2024-09-14T08:35:47.028 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-14T08:35:47.030 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ last_update='2024-09-14 08:35:42' 2024-09-14T08:35:47.030 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2024-09-14 08:35:42' 2024-09-14T08:35:47.030 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:35:47.030 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:35:47.030 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:35:47.030 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state_pattern=up+replaying 2024-09-14T08:35:47.031 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local 'prev_last_update=2024-09-14 08:35:42' 2024-09-14T08:35:47.031 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status 2024-09-14T08:35:47.031 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ CEPH_ARGS= 2024-09-14T08:35:47.031 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-14T08:35:47.093 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ status='OKOKOK142134282mirror.0smithi16419.3.0-4775-g7f24e589trueOK42174290mirror.1smithi16419.3.0-4775-g7f24e589falseOK42214285mirror.2smithi16419.3.0-4775-g7f24e589falseOK42254291mirror.3smithi16419.3.0-4775-g7f24e589falseOKtest1159fdda-fb30-49ff-ba1c-60890cdd16a0up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726302897,"remote_snapshot_timestamp":1726302897,"replay_state":"idle"}42134282mirror.0smithi1642024-09-14 08:35:42cluster29ab74346-7215-4f9b-943c-26e4f257da03up+stoppedlocal image is primary2024-09-14 08:35:41' 2024-09-14T08:35:47.093 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local last_update state 2024-09-14T08:35:47.093 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2024-09-14T08:35:47.136 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ last_update='2024-09-14 08:35:42' 2024-09-14T08:35:47.136 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2024-09-14T08:35:47.138 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ state=up+replaying 2024-09-14T08:35:47.138 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ echo up+replaying 2024-09-14T08:35:47.138 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep up+replaying 2024-09-14T08:35:47.139 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:up+replaying 2024-09-14T08:35:47.140 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:35:47.140 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:35:47.140 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ snap_name=snap1 2024-09-14T08:35:47.140 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_snap cluster2 mirror test snap1 2024-09-14T08:35:47.140 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:35:47.140 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:35:47.140 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:35:47.140 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap1 2024-09-14T08:35:47.140 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap1 --debug-rbd=20 --debug-journaler=20 2024-09-14T08:35:48.882 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.0 2024-09-14T08:35:48.883 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Sent signal 15 2024-09-14T08:35:48.883 INFO:tasks.rbd_mirror_thrash:waiting for 4 secs before reviving daemons 2024-09-14T08:35:48.884 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi164.stderr:2024-09-14T08:35:48.884+0000 7fcbdb268640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 (PID: 36781) UID: 1000 2024-09-14T08:35:49.903 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_started cluster1 mirror test 2024-09-14T08:35:49.903 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:35:49.903 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:35:49.903 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:35:49.903 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2024-09-14T08:35:49.903 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:35:49.903 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:35:49.903 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:35:49.903 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=started 2024-09-14T08:35:49.904 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:35:49.904 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:35:49.904 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:35:50.904 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_replay_state cluster1 mirror test started 2024-09-14T08:35:50.905 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:35:50.905 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:35:50.905 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:35:50.905 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=started 2024-09-14T08:35:50.905 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_result 2024-09-14T08:35:50.905 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=stopped 2024-09-14T08:35:50.906 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2024-09-14T08:35:50.906 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_instance=cluster1 2024-09-14T08:35:50.906 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:35:50.906 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ grep -i state 2024-09-14T08:35:50.906 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T08:35:50.906 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance=cluster1 2024-09-14T08:35:50.907 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local loop_instance 2024-09-14T08:35:50.907 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:35:50.907 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sleep 0 2024-09-14T08:35:50.907 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T08:35:50.907 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ seq 0 3 2024-09-14T08:35:50.909 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:35:50.909 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2024-09-14T08:35:50.910 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:0 2024-09-14T08:35:50.910 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:35:50.910 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:35:50.910 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T08:35:50.910 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:0 2024-09-14T08:35:50.910 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:35:50.910 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:35:50.910 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:35:50.910 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=0 2024-09-14T08:35:50.910 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T08:35:50.910 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:35:50.910 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:35:50.910 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=0 2024-09-14T08:35:50.910 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:35:50.910 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T08:35:50.911 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:0 2024-09-14T08:35:50.911 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:35:50.911 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:35:50.911 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T08:35:50.911 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:0 2024-09-14T08:35:50.911 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:35:50.911 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:35:50.911 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:35:50.911 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:35:50.911 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T08:35:50.911 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:35:50.911 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:35:50.911 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=0 2024-09-14T08:35:50.911 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=0 2024-09-14T08:35:50.911 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T08:35:50.932 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:35:50.932 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:35:50.932 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:35:50.932 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2024-09-14T08:35:51.002 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2024-09-14T08:35:51.005 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:35:51.005 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2024-09-14T08:35:51.005 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:1 2024-09-14T08:35:51.005 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:35:51.005 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:35:51.005 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T08:35:51.005 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:1 2024-09-14T08:35:51.005 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:35:51.005 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:35:51.006 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:35:51.006 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=1 2024-09-14T08:35:51.006 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T08:35:51.006 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:35:51.006 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:35:51.006 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=1 2024-09-14T08:35:51.006 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:35:51.006 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T08:35:51.006 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:1 2024-09-14T08:35:51.006 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:35:51.006 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:35:51.006 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T08:35:51.006 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:1 2024-09-14T08:35:51.006 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:35:51.007 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:35:51.007 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:35:51.007 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:35:51.007 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T08:35:51.007 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:35:51.007 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:35:51.007 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=1 2024-09-14T08:35:51.007 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=1 2024-09-14T08:35:51.007 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T08:35:51.026 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:35:51.026 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:35:51.026 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:35:51.026 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2024-09-14T08:35:51.099 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T08:35:51.099 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:35:51.099 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:35:51.102 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:35:51.102 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2024-09-14T08:35:51.102 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:2 2024-09-14T08:35:51.103 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:35:51.103 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:35:51.103 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-14T08:35:51.103 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:2 2024-09-14T08:35:51.103 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:35:51.103 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:35:51.103 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:35:51.103 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=2 2024-09-14T08:35:51.103 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T08:35:51.103 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:35:51.103 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:35:51.103 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=2 2024-09-14T08:35:51.103 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:35:51.103 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-14T08:35:51.103 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:2 2024-09-14T08:35:51.103 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:35:51.104 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:35:51.104 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T08:35:51.104 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:2 2024-09-14T08:35:51.104 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:35:51.104 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:35:51.104 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:35:51.104 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:35:51.104 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-14T08:35:51.104 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:35:51.104 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:35:51.104 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=2 2024-09-14T08:35:51.104 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=2 2024-09-14T08:35:51.104 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T08:35:51.122 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:35:51.122 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:35:51.122 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:35:51.122 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2024-09-14T08:35:51.194 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T08:35:51.194 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:35:51.194 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:35:51.197 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:35:51.197 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2024-09-14T08:35:51.197 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:3 2024-09-14T08:35:51.197 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:35:51.197 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:35:51.197 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:3 cluster instance 2024-09-14T08:35:51.197 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:3 2024-09-14T08:35:51.197 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:35:51.198 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:35:51.198 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:35:51.198 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=3 2024-09-14T08:35:51.198 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-14T08:35:51.198 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:35:51.198 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:35:51.198 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=3 2024-09-14T08:35:51.198 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=3 2024-09-14T08:35:51.198 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:3 cluster1 2024-09-14T08:35:51.198 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:3 2024-09-14T08:35:51.198 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:35:51.198 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:35:51.198 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-14T08:35:51.198 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:3 2024-09-14T08:35:51.198 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:35:51.199 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:35:51.199 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:35:51.199 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=3 2024-09-14T08:35:51.199 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 3 = cluster1:3 ']' 2024-09-14T08:35:51.199 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:35:51.199 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:35:51.199 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=3 2024-09-14T08:35:51.199 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=3 2024-09-14T08:35:51.199 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-14T08:35:51.215 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T08:35:51.215 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T08:35:51.216 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T08:35:51.216 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2024-09-14T08:35:51.288 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T08:35:51.288 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:35:51.288 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:35:51.289 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:35:51.289 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sleep 1 2024-09-14T08:35:52.290 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T08:35:52.291 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ seq 0 3 2024-09-14T08:35:52.293 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:35:52.293 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2024-09-14T08:35:52.294 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:0 2024-09-14T08:35:52.294 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:35:52.294 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:35:52.294 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T08:35:52.294 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:0 2024-09-14T08:35:52.294 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:35:52.294 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:35:52.294 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:35:52.294 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=0 2024-09-14T08:35:52.294 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T08:35:52.294 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:35:52.294 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:35:52.294 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=0 2024-09-14T08:35:52.294 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:35:52.295 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T08:35:52.295 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:0 2024-09-14T08:35:52.295 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:35:52.295 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:35:52.295 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T08:35:52.295 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:0 2024-09-14T08:35:52.295 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:35:52.295 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:35:52.295 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:35:52.295 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:35:52.295 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T08:35:52.295 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:35:52.295 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:35:52.295 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=0 2024-09-14T08:35:52.295 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=0 2024-09-14T08:35:52.296 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T08:35:52.315 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:35:52.315 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:35:52.316 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:35:52.316 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2024-09-14T08:35:52.384 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2024-09-14T08:35:52.389 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:35:52.389 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2024-09-14T08:35:52.389 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:1 2024-09-14T08:35:52.390 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:35:52.390 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:35:52.390 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T08:35:52.390 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:1 2024-09-14T08:35:52.390 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:35:52.390 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:35:52.391 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:35:52.391 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=1 2024-09-14T08:35:52.391 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T08:35:52.391 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:35:52.392 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:35:52.392 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=1 2024-09-14T08:35:52.392 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:35:52.392 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T08:35:52.392 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:1 2024-09-14T08:35:52.393 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:35:52.393 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:35:52.393 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T08:35:52.393 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:1 2024-09-14T08:35:52.393 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:35:52.393 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:35:52.394 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:35:52.394 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:35:52.394 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T08:35:52.394 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:35:52.394 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:35:52.394 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=1 2024-09-14T08:35:52.395 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=1 2024-09-14T08:35:52.395 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T08:35:52.406 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:35:52.406 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:35:52.407 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:35:52.407 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2024-09-14T08:35:52.481 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T08:35:52.481 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:35:52.481 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:35:52.484 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:35:52.484 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2024-09-14T08:35:52.484 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:2 2024-09-14T08:35:52.484 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:35:52.484 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:35:52.484 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-14T08:35:52.485 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:2 2024-09-14T08:35:52.485 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:35:52.485 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:35:52.485 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:35:52.485 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=2 2024-09-14T08:35:52.485 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T08:35:52.485 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:35:52.485 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:35:52.485 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=2 2024-09-14T08:35:52.485 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:35:52.485 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-14T08:35:52.485 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:2 2024-09-14T08:35:52.485 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:35:52.485 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:35:52.486 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T08:35:52.486 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:2 2024-09-14T08:35:52.486 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:35:52.486 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:35:52.486 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:35:52.486 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:35:52.486 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-14T08:35:52.486 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:35:52.486 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:35:52.486 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=2 2024-09-14T08:35:52.486 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=2 2024-09-14T08:35:52.486 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T08:35:52.502 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:35:52.503 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:35:52.503 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:35:52.503 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2024-09-14T08:35:52.575 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T08:35:52.576 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:35:52.576 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:35:52.579 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:35:52.579 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2024-09-14T08:35:52.579 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:3 2024-09-14T08:35:52.579 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:35:52.579 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:35:52.579 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:3 cluster instance 2024-09-14T08:35:52.580 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:3 2024-09-14T08:35:52.580 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:35:52.580 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:35:52.580 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:35:52.580 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=3 2024-09-14T08:35:52.580 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-14T08:35:52.580 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:35:52.580 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:35:52.580 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=3 2024-09-14T08:35:52.580 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=3 2024-09-14T08:35:52.580 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:3 cluster1 2024-09-14T08:35:52.580 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:3 2024-09-14T08:35:52.581 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:35:52.581 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:35:52.581 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-14T08:35:52.581 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:3 2024-09-14T08:35:52.581 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:35:52.581 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:35:52.581 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:35:52.581 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=3 2024-09-14T08:35:52.581 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 3 = cluster1:3 ']' 2024-09-14T08:35:52.581 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:35:52.581 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:35:52.581 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=3 2024-09-14T08:35:52.581 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=3 2024-09-14T08:35:52.581 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-14T08:35:52.598 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T08:35:52.598 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T08:35:52.598 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T08:35:52.598 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2024-09-14T08:35:52.670 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T08:35:52.671 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:35:52.671 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:35:52.671 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:35:52.672 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sleep 2 2024-09-14T08:35:52.884 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.0 2024-09-14T08:35:52.884 INFO:teuthology.orchestra.run:waiting for 600 2024-09-14T08:35:52.884 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.0 2024-09-14T08:35:52.885 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Restarting daemon 2024-09-14T08:35:52.885 DEBUG:teuthology.orchestra.run.smithi164:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 2024-09-14T08:35:52.887 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Started 2024-09-14T08:35:52.888 INFO:tasks.rbd_mirror_thrash:waiting for 64 secs before thrashing 2024-09-14T08:35:54.673 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T08:35:54.673 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ seq 0 3 2024-09-14T08:35:54.676 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:35:54.676 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2024-09-14T08:35:54.676 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:0 2024-09-14T08:35:54.676 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:35:54.676 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:35:54.677 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T08:35:54.677 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:0 2024-09-14T08:35:54.677 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:35:54.677 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:35:54.677 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:35:54.677 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=0 2024-09-14T08:35:54.677 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T08:35:54.677 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:35:54.677 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:35:54.677 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=0 2024-09-14T08:35:54.677 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:35:54.677 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T08:35:54.677 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:0 2024-09-14T08:35:54.677 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:35:54.678 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:35:54.678 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T08:35:54.678 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:0 2024-09-14T08:35:54.678 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:35:54.678 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:35:54.678 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:35:54.678 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:35:54.678 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T08:35:54.678 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:35:54.678 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:35:54.678 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=0 2024-09-14T08:35:54.678 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=0 2024-09-14T08:35:54.678 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T08:35:54.700 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:35:54.700 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:35:54.700 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:35:54.700 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2024-09-14T08:35:54.779 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T08:35:54.779 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:35:54.780 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:35:54.782 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:35:54.782 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2024-09-14T08:35:54.782 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:1 2024-09-14T08:35:54.782 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:35:54.782 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:35:54.783 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T08:35:54.783 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:1 2024-09-14T08:35:54.783 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:35:54.783 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:35:54.783 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:35:54.783 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=1 2024-09-14T08:35:54.783 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T08:35:54.783 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:35:54.783 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:35:54.783 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=1 2024-09-14T08:35:54.783 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:35:54.783 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T08:35:54.783 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:1 2024-09-14T08:35:54.783 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:35:54.783 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:35:54.783 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T08:35:54.784 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:1 2024-09-14T08:35:54.784 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:35:54.784 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:35:54.784 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:35:54.784 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:35:54.784 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T08:35:54.784 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:35:54.784 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:35:54.784 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=1 2024-09-14T08:35:54.784 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=1 2024-09-14T08:35:54.784 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T08:35:54.802 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:35:54.802 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:35:54.802 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:35:54.802 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2024-09-14T08:35:54.875 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T08:35:54.875 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:35:54.875 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:35:54.878 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:35:54.878 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2024-09-14T08:35:54.878 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:2 2024-09-14T08:35:54.878 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:35:54.878 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:35:54.878 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-14T08:35:54.878 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:2 2024-09-14T08:35:54.878 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:35:54.878 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:35:54.878 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:35:54.879 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=2 2024-09-14T08:35:54.879 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T08:35:54.879 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:35:54.879 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:35:54.879 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=2 2024-09-14T08:35:54.879 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:35:54.879 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-14T08:35:54.879 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:2 2024-09-14T08:35:54.879 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:35:54.879 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:35:54.879 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T08:35:54.879 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:2 2024-09-14T08:35:54.879 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:35:54.879 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:35:54.880 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:35:54.880 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:35:54.880 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-14T08:35:54.880 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:35:54.880 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:35:54.880 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=2 2024-09-14T08:35:54.880 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=2 2024-09-14T08:35:54.880 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T08:35:54.896 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:35:54.896 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:35:54.896 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:35:54.896 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2024-09-14T08:35:54.970 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T08:35:54.970 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:35:54.971 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:35:54.973 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:35:54.974 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2024-09-14T08:35:54.974 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:3 2024-09-14T08:35:54.974 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:35:54.974 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:35:54.974 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:3 cluster instance 2024-09-14T08:35:54.974 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:3 2024-09-14T08:35:54.974 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:35:54.974 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:35:54.974 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:35:54.974 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=3 2024-09-14T08:35:54.974 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-14T08:35:54.974 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:35:54.974 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:35:54.975 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=3 2024-09-14T08:35:54.975 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=3 2024-09-14T08:35:54.975 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:3 cluster1 2024-09-14T08:35:54.975 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:3 2024-09-14T08:35:54.975 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:35:54.975 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:35:54.975 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-14T08:35:54.975 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:3 2024-09-14T08:35:54.975 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:35:54.975 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:35:54.975 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:35:54.975 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=3 2024-09-14T08:35:54.975 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 3 = cluster1:3 ']' 2024-09-14T08:35:54.975 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:35:54.975 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:35:54.976 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=3 2024-09-14T08:35:54.976 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=3 2024-09-14T08:35:54.976 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-14T08:35:54.993 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T08:35:54.993 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T08:35:54.993 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T08:35:54.993 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2024-09-14T08:35:55.067 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T08:35:55.067 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:35:55.067 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:35:55.068 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:35:55.068 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sleep 4 2024-09-14T08:35:59.069 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T08:35:59.070 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ seq 0 3 2024-09-14T08:35:59.072 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:35:59.073 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2024-09-14T08:35:59.073 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:0 2024-09-14T08:35:59.073 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:35:59.073 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:35:59.073 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T08:35:59.073 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:0 2024-09-14T08:35:59.073 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:35:59.073 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:35:59.073 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:35:59.073 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=0 2024-09-14T08:35:59.073 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T08:35:59.073 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:35:59.073 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:35:59.074 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=0 2024-09-14T08:35:59.074 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:35:59.074 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T08:35:59.074 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:0 2024-09-14T08:35:59.074 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:35:59.074 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:35:59.074 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T08:35:59.074 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:0 2024-09-14T08:35:59.074 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:35:59.074 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:35:59.074 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:35:59.074 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:35:59.074 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T08:35:59.074 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:35:59.075 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:35:59.075 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=0 2024-09-14T08:35:59.075 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=0 2024-09-14T08:35:59.075 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T08:35:59.096 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:35:59.096 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:35:59.096 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:35:59.096 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2024-09-14T08:35:59.175 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T08:35:59.175 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:35:59.175 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:35:59.179 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:35:59.179 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2024-09-14T08:35:59.179 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:1 2024-09-14T08:35:59.179 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:35:59.179 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:35:59.179 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T08:35:59.179 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:1 2024-09-14T08:35:59.179 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:35:59.179 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:35:59.179 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:35:59.180 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=1 2024-09-14T08:35:59.180 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T08:35:59.180 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:35:59.180 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:35:59.180 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=1 2024-09-14T08:35:59.180 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:35:59.180 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T08:35:59.180 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:1 2024-09-14T08:35:59.180 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:35:59.180 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:35:59.180 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T08:35:59.181 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:1 2024-09-14T08:35:59.181 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:35:59.181 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:35:59.181 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:35:59.181 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:35:59.181 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T08:35:59.181 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:35:59.181 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:35:59.181 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=1 2024-09-14T08:35:59.181 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=1 2024-09-14T08:35:59.181 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T08:35:59.199 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:35:59.199 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:35:59.199 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:35:59.199 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2024-09-14T08:35:59.272 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T08:35:59.272 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:35:59.272 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:35:59.275 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:35:59.276 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2024-09-14T08:35:59.276 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:2 2024-09-14T08:35:59.276 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:35:59.276 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:35:59.276 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-14T08:35:59.276 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:2 2024-09-14T08:35:59.276 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:35:59.276 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:35:59.276 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:35:59.276 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=2 2024-09-14T08:35:59.276 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T08:35:59.276 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:35:59.276 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:35:59.276 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=2 2024-09-14T08:35:59.277 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:35:59.277 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-14T08:35:59.277 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:2 2024-09-14T08:35:59.277 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:35:59.277 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:35:59.277 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T08:35:59.277 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:2 2024-09-14T08:35:59.277 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:35:59.277 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:35:59.277 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:35:59.277 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:35:59.277 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-14T08:35:59.278 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:35:59.278 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:35:59.278 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=2 2024-09-14T08:35:59.278 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=2 2024-09-14T08:35:59.278 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T08:35:59.294 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:35:59.295 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:35:59.295 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:35:59.295 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2024-09-14T08:35:59.367 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T08:35:59.367 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:35:59.367 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:35:59.370 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:35:59.370 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2024-09-14T08:35:59.370 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:3 2024-09-14T08:35:59.370 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:35:59.370 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:35:59.370 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:3 cluster instance 2024-09-14T08:35:59.371 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:3 2024-09-14T08:35:59.371 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:35:59.371 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:35:59.371 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:35:59.371 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=3 2024-09-14T08:35:59.371 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-14T08:35:59.371 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:35:59.371 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:35:59.371 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=3 2024-09-14T08:35:59.371 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=3 2024-09-14T08:35:59.371 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:3 cluster1 2024-09-14T08:35:59.371 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:3 2024-09-14T08:35:59.371 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:35:59.372 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:35:59.372 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-14T08:35:59.372 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:3 2024-09-14T08:35:59.372 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:35:59.372 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:35:59.372 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:35:59.372 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=3 2024-09-14T08:35:59.372 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 3 = cluster1:3 ']' 2024-09-14T08:35:59.372 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:35:59.372 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:35:59.372 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=3 2024-09-14T08:35:59.372 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=3 2024-09-14T08:35:59.372 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-14T08:35:59.388 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T08:35:59.388 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T08:35:59.388 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T08:35:59.388 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2024-09-14T08:35:59.461 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T08:35:59.461 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:35:59.461 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:35:59.462 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:35:59.462 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sleep 8 2024-09-14T08:36:07.463 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T08:36:07.463 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ seq 0 3 2024-09-14T08:36:07.466 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:36:07.466 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2024-09-14T08:36:07.466 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:0 2024-09-14T08:36:07.466 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:36:07.466 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:36:07.466 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T08:36:07.467 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:0 2024-09-14T08:36:07.467 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:36:07.467 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:36:07.467 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:36:07.467 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=0 2024-09-14T08:36:07.467 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T08:36:07.467 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:36:07.467 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:36:07.467 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=0 2024-09-14T08:36:07.467 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:36:07.467 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T08:36:07.467 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:0 2024-09-14T08:36:07.467 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:36:07.467 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:36:07.468 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T08:36:07.468 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:0 2024-09-14T08:36:07.468 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:36:07.468 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:36:07.468 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:36:07.468 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:36:07.468 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T08:36:07.468 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:36:07.468 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:36:07.468 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=0 2024-09-14T08:36:07.468 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=0 2024-09-14T08:36:07.468 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T08:36:07.488 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:36:07.489 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:36:07.489 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:36:07.489 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2024-09-14T08:36:07.568 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T08:36:07.568 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:36:07.568 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:36:07.571 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:36:07.571 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2024-09-14T08:36:07.571 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:1 2024-09-14T08:36:07.571 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:36:07.571 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:36:07.571 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T08:36:07.571 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:1 2024-09-14T08:36:07.572 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:36:07.572 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:36:07.572 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:36:07.572 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=1 2024-09-14T08:36:07.572 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T08:36:07.572 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:36:07.572 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:36:07.572 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=1 2024-09-14T08:36:07.572 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:36:07.572 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T08:36:07.572 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:1 2024-09-14T08:36:07.572 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:36:07.572 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:36:07.572 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T08:36:07.572 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:1 2024-09-14T08:36:07.573 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:36:07.573 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:36:07.573 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:36:07.573 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:36:07.573 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T08:36:07.573 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:36:07.573 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:36:07.573 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=1 2024-09-14T08:36:07.573 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=1 2024-09-14T08:36:07.573 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T08:36:07.591 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:36:07.591 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:36:07.591 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:36:07.591 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2024-09-14T08:36:07.663 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T08:36:07.663 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:36:07.663 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:36:07.666 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:36:07.666 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2024-09-14T08:36:07.666 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:2 2024-09-14T08:36:07.666 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:36:07.666 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:36:07.666 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-14T08:36:07.666 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:2 2024-09-14T08:36:07.667 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:36:07.667 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:36:07.667 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:36:07.667 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=2 2024-09-14T08:36:07.667 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T08:36:07.667 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:36:07.667 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:36:07.667 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=2 2024-09-14T08:36:07.667 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:36:07.667 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-14T08:36:07.668 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:2 2024-09-14T08:36:07.668 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:36:07.668 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:36:07.668 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T08:36:07.668 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:2 2024-09-14T08:36:07.668 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:36:07.669 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:36:07.669 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:36:07.669 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:36:07.669 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-14T08:36:07.669 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:36:07.669 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:36:07.669 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=2 2024-09-14T08:36:07.669 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=2 2024-09-14T08:36:07.669 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T08:36:07.684 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:36:07.684 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:36:07.684 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:36:07.684 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2024-09-14T08:36:07.758 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T08:36:07.758 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:36:07.758 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:36:07.761 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:36:07.761 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2024-09-14T08:36:07.761 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:3 2024-09-14T08:36:07.762 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:36:07.762 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:36:07.762 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:3 cluster instance 2024-09-14T08:36:07.762 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:3 2024-09-14T08:36:07.762 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:36:07.762 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:36:07.762 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:36:07.762 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=3 2024-09-14T08:36:07.762 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-14T08:36:07.762 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:36:07.762 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:36:07.762 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=3 2024-09-14T08:36:07.762 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=3 2024-09-14T08:36:07.762 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:3 cluster1 2024-09-14T08:36:07.763 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:3 2024-09-14T08:36:07.763 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:36:07.763 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:36:07.763 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-14T08:36:07.763 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:3 2024-09-14T08:36:07.763 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:36:07.763 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:36:07.763 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:36:07.763 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=3 2024-09-14T08:36:07.763 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 3 = cluster1:3 ']' 2024-09-14T08:36:07.763 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:36:07.763 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:36:07.764 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=3 2024-09-14T08:36:07.764 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=3 2024-09-14T08:36:07.764 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-14T08:36:07.780 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T08:36:07.780 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T08:36:07.780 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T08:36:07.780 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2024-09-14T08:36:07.851 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T08:36:07.851 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:36:07.851 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:36:07.852 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:36:07.852 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sleep 8 2024-09-14T08:36:15.853 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T08:36:15.854 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ seq 0 3 2024-09-14T08:36:15.856 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:36:15.857 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2024-09-14T08:36:15.857 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:0 2024-09-14T08:36:15.857 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:36:15.857 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:36:15.857 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T08:36:15.857 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:0 2024-09-14T08:36:15.857 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:36:15.857 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:36:15.857 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:36:15.857 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=0 2024-09-14T08:36:15.857 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T08:36:15.857 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:36:15.857 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:36:15.857 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=0 2024-09-14T08:36:15.858 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:36:15.858 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T08:36:15.858 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:0 2024-09-14T08:36:15.858 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:36:15.858 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:36:15.858 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T08:36:15.858 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:0 2024-09-14T08:36:15.858 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:36:15.858 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:36:15.858 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:36:15.858 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:36:15.858 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T08:36:15.858 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:36:15.859 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:36:15.859 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=0 2024-09-14T08:36:15.859 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=0 2024-09-14T08:36:15.859 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T08:36:15.880 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:36:15.880 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:36:15.880 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:36:15.880 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2024-09-14T08:36:15.960 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T08:36:15.960 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:36:15.960 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:36:15.963 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:36:15.963 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2024-09-14T08:36:15.963 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:1 2024-09-14T08:36:15.963 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:36:15.964 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:36:15.964 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T08:36:15.964 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:1 2024-09-14T08:36:15.964 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:36:15.964 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:36:15.964 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:36:15.964 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=1 2024-09-14T08:36:15.964 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T08:36:15.964 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:36:15.964 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:36:15.964 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=1 2024-09-14T08:36:15.964 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:36:15.965 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T08:36:15.965 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:1 2024-09-14T08:36:15.965 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:36:15.965 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:36:15.965 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T08:36:15.965 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:1 2024-09-14T08:36:15.965 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:36:15.965 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:36:15.965 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:36:15.965 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:36:15.965 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T08:36:15.965 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:36:15.965 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:36:15.966 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=1 2024-09-14T08:36:15.966 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=1 2024-09-14T08:36:15.966 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T08:36:15.982 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:36:15.982 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:36:15.983 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:36:15.983 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2024-09-14T08:36:16.057 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T08:36:16.057 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:36:16.057 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:36:16.060 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:36:16.061 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2024-09-14T08:36:16.061 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:2 2024-09-14T08:36:16.061 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:36:16.061 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:36:16.061 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-14T08:36:16.061 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:2 2024-09-14T08:36:16.061 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:36:16.061 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:36:16.061 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:36:16.061 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=2 2024-09-14T08:36:16.061 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T08:36:16.061 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:36:16.061 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:36:16.061 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=2 2024-09-14T08:36:16.062 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:36:16.062 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-14T08:36:16.062 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:2 2024-09-14T08:36:16.062 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:36:16.062 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:36:16.062 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T08:36:16.062 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:2 2024-09-14T08:36:16.062 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:36:16.062 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:36:16.062 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:36:16.062 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:36:16.063 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-14T08:36:16.063 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:36:16.063 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:36:16.063 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=2 2024-09-14T08:36:16.063 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=2 2024-09-14T08:36:16.063 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T08:36:16.078 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:36:16.079 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:36:16.079 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:36:16.079 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2024-09-14T08:36:16.151 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T08:36:16.151 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:36:16.151 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:36:16.154 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:36:16.155 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2024-09-14T08:36:16.155 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:3 2024-09-14T08:36:16.155 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:36:16.155 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:36:16.155 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:3 cluster instance 2024-09-14T08:36:16.155 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:3 2024-09-14T08:36:16.155 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:36:16.155 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:36:16.155 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:36:16.155 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=3 2024-09-14T08:36:16.155 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-14T08:36:16.155 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:36:16.156 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:36:16.156 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=3 2024-09-14T08:36:16.156 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=3 2024-09-14T08:36:16.156 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:3 cluster1 2024-09-14T08:36:16.156 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:3 2024-09-14T08:36:16.156 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:36:16.156 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:36:16.156 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-14T08:36:16.156 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:3 2024-09-14T08:36:16.156 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:36:16.156 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:36:16.156 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:36:16.156 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=3 2024-09-14T08:36:16.157 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 3 = cluster1:3 ']' 2024-09-14T08:36:16.157 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:36:16.157 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:36:16.157 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=3 2024-09-14T08:36:16.157 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=3 2024-09-14T08:36:16.157 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-14T08:36:16.173 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T08:36:16.173 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T08:36:16.173 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T08:36:16.173 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2024-09-14T08:36:16.245 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T08:36:16.245 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:36:16.245 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:36:16.246 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:36:16.246 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sleep 8 2024-09-14T08:36:24.247 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T08:36:24.248 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ seq 0 3 2024-09-14T08:36:24.251 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:36:24.251 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2024-09-14T08:36:24.251 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:0 2024-09-14T08:36:24.251 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:36:24.252 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:36:24.252 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T08:36:24.252 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:0 2024-09-14T08:36:24.252 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:36:24.252 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:36:24.252 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:36:24.252 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=0 2024-09-14T08:36:24.252 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T08:36:24.252 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:36:24.252 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:36:24.252 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=0 2024-09-14T08:36:24.253 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:36:24.253 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T08:36:24.253 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:0 2024-09-14T08:36:24.253 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:36:24.253 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:36:24.253 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T08:36:24.253 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:0 2024-09-14T08:36:24.253 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:36:24.253 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:36:24.253 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:36:24.254 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:36:24.254 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T08:36:24.254 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:36:24.254 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:36:24.254 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=0 2024-09-14T08:36:24.254 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=0 2024-09-14T08:36:24.254 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T08:36:24.274 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:36:24.274 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:36:24.274 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:36:24.274 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2024-09-14T08:36:24.353 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T08:36:24.353 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:36:24.353 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:36:24.356 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:36:24.356 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2024-09-14T08:36:24.357 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:1 2024-09-14T08:36:24.357 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:36:24.357 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:36:24.357 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T08:36:24.357 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:1 2024-09-14T08:36:24.357 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:36:24.357 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:36:24.357 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:36:24.357 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=1 2024-09-14T08:36:24.358 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T08:36:24.358 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:36:24.358 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:36:24.358 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=1 2024-09-14T08:36:24.358 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:36:24.358 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T08:36:24.358 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:1 2024-09-14T08:36:24.358 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:36:24.358 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:36:24.358 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T08:36:24.358 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:1 2024-09-14T08:36:24.358 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:36:24.358 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:36:24.358 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:36:24.358 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:36:24.359 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T08:36:24.359 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:36:24.359 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:36:24.359 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=1 2024-09-14T08:36:24.359 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=1 2024-09-14T08:36:24.359 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T08:36:24.375 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:36:24.375 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:36:24.375 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:36:24.375 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2024-09-14T08:36:24.449 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T08:36:24.449 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:36:24.450 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:36:24.453 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:36:24.453 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2024-09-14T08:36:24.453 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:2 2024-09-14T08:36:24.453 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:36:24.453 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:36:24.453 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-14T08:36:24.453 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:2 2024-09-14T08:36:24.453 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:36:24.453 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:36:24.453 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:36:24.453 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=2 2024-09-14T08:36:24.453 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T08:36:24.453 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:36:24.454 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:36:24.454 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=2 2024-09-14T08:36:24.454 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:36:24.454 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-14T08:36:24.454 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:2 2024-09-14T08:36:24.454 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:36:24.454 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:36:24.454 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T08:36:24.454 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:2 2024-09-14T08:36:24.455 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:36:24.455 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:36:24.455 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:36:24.455 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:36:24.455 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-14T08:36:24.455 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:36:24.455 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:36:24.455 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=2 2024-09-14T08:36:24.455 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=2 2024-09-14T08:36:24.455 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T08:36:24.471 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:36:24.471 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:36:24.471 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:36:24.471 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2024-09-14T08:36:24.552 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ return 0 2024-09-14T08:36:24.552 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T08:36:24.552 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ echo ' "state": "Replaying"' 2024-09-14T08:36:24.552 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep -i Replaying 2024-09-14T08:36:24.553 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: "state": "Replaying" 2024-09-14T08:36:24.554 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_state=started 2024-09-14T08:36:24.554 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test started = started 2024-09-14T08:36:24.554 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:36:24.555 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2024-09-14T08:36:24.555 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:36:24.555 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:36:24.555 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:36:24.555 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:36:24.555 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = journal ']' 2024-09-14T08:36:24.555 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = snapshot ']' 2024-09-14T08:36:24.555 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test 2024-09-14T08:36:24.555 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:36:24.555 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:36:24.555 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:36:24.555 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:36:24.556 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster2-mirror-test.status 2024-09-14T08:36:24.556 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster2-mirror-test.status 2024-09-14T08:36:24.556 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:36:24.557 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-test.status 2024-09-14T08:36:24.557 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster1-mirror-test.status 2024-09-14T08:36:24.557 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster1-mirror-test.status 2024-09-14T08:36:24.557 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:36:24.558 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_status_log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:36:24.558 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ mirror_image_snapshot cluster2 mirror test 2024-09-14T08:36:24.558 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:36:24.558 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:36:24.558 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:36:24.558 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2024-09-14T08:36:24.942 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Snapshot ID: 5 2024-09-14T08:36:24.947 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.status 2024-09-14T08:36:24.947 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:36:24.947 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:36:24.947 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:36:24.947 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-test.status 2024-09-14T08:36:24.947 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2024-09-14T08:36:24.947 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:36:24.993 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-14T08:36:24.994 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snapshot_id=5 2024-09-14T08:36:24.994 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:36:24.995 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:36:24.995 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.2 2024-09-14T08:36:25.196 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:36:25.196 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:36:25.196 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:36:25.196 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:36:25.196 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:36:25.196 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-09-14T08:36:25.196 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:36:25.246 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:36:25.248 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=3 2024-09-14T08:36:25.248 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 5 = 3 2024-09-14T08:36:25.248 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:36:25.248 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.4 2024-09-14T08:36:25.649 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:36:25.649 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:36:25.649 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:36:25.649 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:36:25.650 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:36:25.650 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-09-14T08:36:25.650 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:36:25.699 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:36:25.700 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:36:25.701 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:36:25.701 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:36:25.701 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:36:25.701 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:36:25.702 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 5 = '' 2024-09-14T08:36:25.702 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:36:25.702 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.8 2024-09-14T08:36:26.503 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:36:26.503 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:36:26.503 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:36:26.503 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:36:26.504 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:36:26.504 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-09-14T08:36:26.504 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:36:26.677 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:36:26.677 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:36:26.678 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:36:26.678 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:36:26.678 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:36:26.679 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:36:26.679 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 5 = '' 2024-09-14T08:36:26.679 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:36:26.679 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1.6 2024-09-14T08:36:27.342 INFO:tasks.ceph.cluster1.osd.0.smithi135.stderr:2024-09-14T08:36:27.338+0000 7f31e68c4640 -1 reset not still connected to 0x5623cd150340 2024-09-14T08:36:28.280 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:36:28.281 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:36:28.281 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:36:28.281 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:36:28.281 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:36:28.281 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-09-14T08:36:28.281 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:36:28.331 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:36:28.333 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=5 2024-09-14T08:36:28.333 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 5 = 5 2024-09-14T08:36:28.333 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:36:28.333 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snap_present cluster1 mirror test snap1 2024-09-14T08:36:28.333 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:36:28.333 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:36:28.333 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:36:28.333 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap1 2024-09-14T08:36:28.333 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:36:28.334 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T08:36:28.334 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:36:29.334 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 info mirror/test@snap1 2024-09-14T08:36:29.383 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'test': 2024-09-14T08:36:29.384 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 512 MiB in 128 objects 2024-09-14T08:36:29.384 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:36:29.384 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 2 2024-09-14T08:36:29.384 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 10ba6324d73b 2024-09-14T08:36:29.384 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.10ba6324d73b 2024-09-14T08:36:29.384 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:36:29.384 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering, non-primary 2024-09-14T08:36:29.384 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:36:29.384 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:36:29.384 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:35:14 2024 2024-09-14T08:36:29.384 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:35:14 2024 2024-09-14T08:36:29.384 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:35:14 2024 2024-09-14T08:36:29.384 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: protected: False 2024-09-14T08:36:29.385 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:36:29.385 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:36:29.385 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: 1159fdda-fb30-49ff-ba1c-60890cdd16a0 2024-09-14T08:36:29.385 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: false 2024-09-14T08:36:29.388 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:36:29.388 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n '' ']' 2024-09-14T08:36:29.388 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ compare_image_snaps mirror test snap1 2024-09-14T08:36:29.388 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:36:29.388 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:36:29.388 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap1 2024-09-14T08:36:29.388 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local ret=0 2024-09-14T08:36:29.388 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local rmt_export=/tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export 2024-09-14T08:36:29.388 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local loc_export=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:36:29.389 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:36:29.389 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror export test@snap1 /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export 2024-09-14T08:36:29.991 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T08:36:30.001 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 -p mirror export test@snap1 /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:36:30.804 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T08:36:30.813 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cmp /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:36:31.023 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:36:31.136 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:36:31.137 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ clean_snap_name=snap1-clean 2024-09-14T08:36:31.137 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_snap cluster2 mirror test snap1-clean 2024-09-14T08:36:31.137 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:36:31.137 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:36:31.137 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:36:31.137 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap1-clean 2024-09-14T08:36:31.137 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap1-clean --debug-rbd=20 --debug-journaler=20 2024-09-14T08:36:31.949 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 10` 2024-09-14T08:36:31.949 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ stress_write_image cluster2 mirror test 2024-09-14T08:36:31.949 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:36:31.949 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:36:31.950 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:36:31.950 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T08:36:31.951 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local duration=12 2024-09-14T08:36:31.951 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set +e 2024-09-14T08:36:31.951 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ timeout 12s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2024-09-14T08:36:32.333 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:100 IOs, 97557504 bytes 2024-09-14T08:36:32.484 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:200 IOs, 132741120 bytes 2024-09-14T08:36:32.621 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:300 IOs, 164621312 bytes 2024-09-14T08:36:32.758 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:400 IOs, 213185536 bytes 2024-09-14T08:36:32.888 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.2975215037644362) > weight (0.25) 2024-09-14T08:36:32.888 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.6973460045656757) > weight (0.25) 2024-09-14T08:36:32.888 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.7252149756349974) > weight (0.25) 2024-09-14T08:36:32.888 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.3 with skip (0.5612217130558441) > weight (0.25) 2024-09-14T08:36:32.888 INFO:tasks.rbd_mirror_thrash:waiting for 79 secs before thrashing 2024-09-14T08:36:32.992 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:500 IOs, 289229824 bytes 2024-09-14T08:36:33.141 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:600 IOs, 321343488 bytes 2024-09-14T08:36:33.248 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:700 IOs, 340021248 bytes 2024-09-14T08:36:33.403 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:800 IOs, 396918784 bytes 2024-09-14T08:36:33.584 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:900 IOs, 450161664 bytes 2024-09-14T08:36:33.757 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1000 IOs, 495893504 bytes 2024-09-14T08:36:34.070 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1100 IOs, 604308480 bytes 2024-09-14T08:36:34.242 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1200 IOs, 673196032 bytes 2024-09-14T08:36:34.361 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1300 IOs, 687296512 bytes 2024-09-14T08:36:34.607 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1400 IOs, 773142528 bytes 2024-09-14T08:36:34.742 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1500 IOs, 808457216 bytes 2024-09-14T08:36:34.855 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1600 IOs, 835873792 bytes 2024-09-14T08:36:34.964 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1700 IOs, 880144384 bytes 2024-09-14T08:36:35.171 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1800 IOs, 925970432 bytes 2024-09-14T08:36:35.509 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1900 IOs, 1052479488 bytes 2024-09-14T08:36:35.707 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2000 IOs, 1117632512 bytes 2024-09-14T08:36:35.795 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2100 IOs, 1126002688 bytes 2024-09-14T08:36:35.916 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2200 IOs, 1168816128 bytes 2024-09-14T08:36:36.076 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2300 IOs, 1202057216 bytes 2024-09-14T08:36:36.183 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2400 IOs, 1230357504 bytes 2024-09-14T08:36:36.253 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2500 IOs, 1233381376 bytes 2024-09-14T08:36:36.370 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2600 IOs, 1262129152 bytes 2024-09-14T08:36:36.535 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2700 IOs, 1312204800 bytes 2024-09-14T08:36:36.673 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2800 IOs, 1357447168 bytes 2024-09-14T08:36:36.844 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2900 IOs, 1399001088 bytes 2024-09-14T08:36:37.033 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3000 IOs, 1464485888 bytes 2024-09-14T08:36:37.125 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3100 IOs, 1466651648 bytes 2024-09-14T08:36:37.327 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3200 IOs, 1534458880 bytes 2024-09-14T08:36:37.502 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3300 IOs, 1578815488 bytes 2024-09-14T08:36:37.588 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3400 IOs, 1593370624 bytes 2024-09-14T08:36:37.832 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3500 IOs, 1690266624 bytes 2024-09-14T08:36:38.009 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3600 IOs, 1732495360 bytes 2024-09-14T08:36:38.296 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3700 IOs, 1835000832 bytes 2024-09-14T08:36:38.372 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3800 IOs, 1842657280 bytes 2024-09-14T08:36:38.511 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3900 IOs, 1899409408 bytes 2024-09-14T08:36:38.602 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4000 IOs, 1900012544 bytes 2024-09-14T08:36:38.711 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4100 IOs, 1924226048 bytes 2024-09-14T08:36:38.792 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4200 IOs, 1939092480 bytes 2024-09-14T08:36:38.875 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4300 IOs, 1962101760 bytes 2024-09-14T08:36:38.997 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4400 IOs, 1981785088 bytes 2024-09-14T08:36:39.076 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4500 IOs, 1987069952 bytes 2024-09-14T08:36:39.355 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4600 IOs, 2094657536 bytes 2024-09-14T08:36:39.462 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4700 IOs, 2114232320 bytes 2024-09-14T08:36:39.539 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4800 IOs, 2119471104 bytes 2024-09-14T08:36:39.641 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4900 IOs, 2142945280 bytes 2024-09-14T08:36:39.754 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5000 IOs, 2173803520 bytes 2024-09-14T08:36:39.879 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5100 IOs, 2199321600 bytes 2024-09-14T08:36:39.979 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5200 IOs, 2217783296 bytes 2024-09-14T08:36:40.205 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5300 IOs, 2308360192 bytes 2024-09-14T08:36:40.346 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5400 IOs, 2338970624 bytes 2024-09-14T08:36:40.494 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5500 IOs, 2383489024 bytes 2024-09-14T08:36:40.711 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5600 IOs, 2451892224 bytes 2024-09-14T08:36:40.960 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5700 IOs, 2528990208 bytes 2024-09-14T08:36:41.171 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5800 IOs, 2599294976 bytes 2024-09-14T08:36:41.308 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5900 IOs, 2625837056 bytes 2024-09-14T08:36:41.366 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6000 IOs, 2636401664 bytes 2024-09-14T08:36:41.563 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6100 IOs, 2693010432 bytes 2024-09-14T08:36:41.712 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6200 IOs, 2738532352 bytes 2024-09-14T08:36:41.830 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6300 IOs, 2766370816 bytes 2024-09-14T08:36:41.963 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6400 IOs, 2792876032 bytes 2024-09-14T08:36:42.133 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6500 IOs, 2859208704 bytes 2024-09-14T08:36:42.254 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6600 IOs, 2865316864 bytes 2024-09-14T08:36:42.536 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6700 IOs, 2967993344 bytes 2024-09-14T08:36:42.708 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6800 IOs, 3012750336 bytes 2024-09-14T08:36:42.800 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6900 IOs, 3031351296 bytes 2024-09-14T08:36:42.944 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:7000 IOs, 3073368064 bytes 2024-09-14T08:36:43.321 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:7100 IOs, 3212914688 bytes 2024-09-14T08:36:43.413 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:7200 IOs, 3228224512 bytes 2024-09-14T08:36:43.588 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:7300 IOs, 3284925440 bytes 2024-09-14T08:36:43.768 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:7400 IOs, 3338299392 bytes 2024-09-14T08:36:43.957 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ error_code=124 2024-09-14T08:36:43.957 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set -e 2024-09-14T08:36:43.957 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T08:36:43.957 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:36:43.957 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2024-09-14T08:36:43.957 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:36:43.957 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:36:43.957 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:36:43.957 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state_pattern=up+replaying 2024-09-14T08:36:43.957 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local description_pattern= 2024-09-14T08:36:43.957 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local service_pattern= 2024-09-14T08:36:43.957 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:36:43.958 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:36:44.958 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2024-09-14T08:36:44.958 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:36:44.958 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:36:44.958 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:36:44.958 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state_pattern=up+replaying 2024-09-14T08:36:44.959 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local description_pattern= 2024-09-14T08:36:44.959 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local service_pattern= 2024-09-14T08:36:44.959 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status 2024-09-14T08:36:44.959 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ CEPH_ARGS= 2024-09-14T08:36:44.959 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2024-09-14T08:36:45.015 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ status='test: 2024-09-14T08:36:45.015 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: global_id: 1159fdda-fb30-49ff-ba1c-60890cdd16a0 2024-09-14T08:36:45.015 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: state: up+replaying 2024-09-14T08:36:45.015 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: description: replaying, {"bytes_per_second":26304665.6,"bytes_per_snapshot":526093312.0,"last_snapshot_bytes":526093312,"last_snapshot_sync_seconds":1,"local_snapshot_timestamp":1726302984,"remote_snapshot_timestamp":1726302984,"replay_state":"idle"} 2024-09-14T08:36:45.015 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: service: mirror.2 on smithi164 2024-09-14T08:36:45.015 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: last_update: 2024-09-14 08:36:42 2024-09-14T08:36:45.016 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: peer_sites: 2024-09-14T08:36:45.016 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: name: cluster2 2024-09-14T08:36:45.016 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: state: up+stopped 2024-09-14T08:36:45.016 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: description: local image is primary 2024-09-14T08:36:45.016 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: last_update: 2024-09-14 08:36:41' 2024-09-14T08:36:45.016 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep '^ state: .*up+replaying' 2024-09-14T08:36:45.016 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: state: up+replaying 2024-09-14T08:36:45.017 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep '^ description: .*' 2024-09-14T08:36:45.017 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: description: replaying, {"bytes_per_second":26304665.6,"bytes_per_snapshot":526093312.0,"last_snapshot_bytes":526093312,"last_snapshot_sync_seconds":1,"local_snapshot_timestamp":1726302984,"remote_snapshot_timestamp":1726302984,"replay_state":"idle"} 2024-09-14T08:36:45.018 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n '' ']' 2024-09-14T08:36:45.018 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ echo up+replaying 2024-09-14T08:36:45.018 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep '^up+' 2024-09-14T08:36:45.018 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:up+replaying 2024-09-14T08:36:45.019 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep 'service: *mirror..* on ' 2024-09-14T08:36:45.020 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: service: mirror.2 on smithi164 2024-09-14T08:36:45.020 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local last_update 2024-09-14T08:36:45.020 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-14T08:36:45.022 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ last_update='2024-09-14 08:36:42' 2024-09-14T08:36:45.022 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2024-09-14 08:36:42' 2024-09-14T08:36:45.022 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:36:45.022 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:36:45.023 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:36:45.023 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state_pattern=up+replaying 2024-09-14T08:36:45.023 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local 'prev_last_update=2024-09-14 08:36:42' 2024-09-14T08:36:45.023 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status 2024-09-14T08:36:45.023 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ CEPH_ARGS= 2024-09-14T08:36:45.023 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-14T08:36:45.082 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ status='OKOKOK142174290mirror.1smithi16419.3.0-4775-g7f24e589trueOK42214285mirror.2smithi16419.3.0-4775-g7f24e589falseOK42254291mirror.3smithi16419.3.0-4775-g7f24e589falseOK43274330mirror.0smithi16419.3.0-4775-g7f24e589falseOKtest1159fdda-fb30-49ff-ba1c-60890cdd16a0up+replayingreplaying, {"bytes_per_second":26304665.6,"bytes_per_snapshot":526093312.0,"last_snapshot_bytes":526093312,"last_snapshot_sync_seconds":1,"local_snapshot_timestamp":1726302984,"remote_snapshot_timestamp":1726302984,"replay_state":"idle"}42214285mirror.2smithi1642024-09-14 08:36:42cluster29ab74346-7215-4f9b-943c-26e4f257da03up+stoppedlocal image is primary2024-09-14 08:36:41' 2024-09-14T08:36:45.082 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local last_update state 2024-09-14T08:36:45.083 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2024-09-14T08:36:45.084 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ last_update='2024-09-14 08:36:42' 2024-09-14T08:36:45.084 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2024-09-14T08:36:45.086 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ state=up+replaying 2024-09-14T08:36:45.086 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ echo up+replaying 2024-09-14T08:36:45.086 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep up+replaying 2024-09-14T08:36:45.087 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:up+replaying 2024-09-14T08:36:45.088 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:36:45.088 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:36:45.088 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ snap_name=snap2 2024-09-14T08:36:45.088 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_snap cluster2 mirror test snap2 2024-09-14T08:36:45.088 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:36:45.088 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:36:45.088 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:36:45.089 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap2 2024-09-14T08:36:45.089 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap2 --debug-rbd=20 --debug-journaler=20 2024-09-14T08:36:56.888 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.8805669642213382) > weight (0.25) 2024-09-14T08:36:56.889 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.540805694185043) > weight (0.25) 2024-09-14T08:36:56.889 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.7326998183307946) > weight (0.25) 2024-09-14T08:36:56.889 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.3 2024-09-14T08:36:56.890 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Sent signal 15 2024-09-14T08:36:56.890 INFO:tasks.rbd_mirror_thrash:waiting for 9 secs before reviving daemons 2024-09-14T08:36:56.891 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi164.stderr:2024-09-14T08:36:56.891+0000 7f10cf3ad640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.3 (PID: 36831) UID: 1000 2024-09-14T08:37:05.891 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.3 2024-09-14T08:37:05.891 INFO:teuthology.orchestra.run:waiting for 600 2024-09-14T08:37:05.892 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.3 2024-09-14T08:37:05.892 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Restarting daemon 2024-09-14T08:37:05.892 DEBUG:teuthology.orchestra.run.smithi164:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.3 2024-09-14T08:37:05.897 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Started 2024-09-14T08:37:05.897 INFO:tasks.rbd_mirror_thrash:waiting for 62 secs before thrashing 2024-09-14T08:37:12.995 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_started cluster1 mirror test 2024-09-14T08:37:12.995 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:37:12.996 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:37:12.996 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:37:12.996 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2024-09-14T08:37:12.996 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:37:12.996 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:37:12.996 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:37:12.996 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=started 2024-09-14T08:37:12.996 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:37:12.996 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:37:12.997 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:37:13.996 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_replay_state cluster1 mirror test started 2024-09-14T08:37:13.996 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:37:13.996 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:37:13.997 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:37:13.997 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=started 2024-09-14T08:37:13.997 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_result 2024-09-14T08:37:13.997 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=stopped 2024-09-14T08:37:13.998 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2024-09-14T08:37:13.998 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_instance=cluster1 2024-09-14T08:37:13.998 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:37:13.998 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ grep -i state 2024-09-14T08:37:13.998 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T08:37:13.998 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance=cluster1 2024-09-14T08:37:13.998 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local loop_instance 2024-09-14T08:37:13.998 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:37:13.999 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sleep 0 2024-09-14T08:37:13.999 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T08:37:13.999 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ seq 0 3 2024-09-14T08:37:14.000 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:37:14.000 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2024-09-14T08:37:14.000 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:0 2024-09-14T08:37:14.001 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:37:14.001 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:37:14.001 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T08:37:14.001 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:0 2024-09-14T08:37:14.001 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:37:14.001 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:37:14.001 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:37:14.001 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=0 2024-09-14T08:37:14.001 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T08:37:14.001 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:37:14.001 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:37:14.001 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=0 2024-09-14T08:37:14.001 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:37:14.002 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T08:37:14.002 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:0 2024-09-14T08:37:14.002 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:37:14.002 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:37:14.003 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T08:37:14.003 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:0 2024-09-14T08:37:14.003 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:37:14.003 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:37:14.003 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:37:14.003 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:37:14.003 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T08:37:14.003 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:37:14.004 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:37:14.004 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=0 2024-09-14T08:37:14.004 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=0 2024-09-14T08:37:14.004 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T08:37:14.021 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:37:14.021 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:37:14.021 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:37:14.021 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2024-09-14T08:37:14.098 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T08:37:14.098 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:37:14.098 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:37:14.101 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:37:14.101 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2024-09-14T08:37:14.101 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:1 2024-09-14T08:37:14.101 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:37:14.101 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:37:14.101 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T08:37:14.101 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:1 2024-09-14T08:37:14.101 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:37:14.102 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:37:14.102 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:37:14.102 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=1 2024-09-14T08:37:14.102 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T08:37:14.102 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:37:14.102 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:37:14.102 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=1 2024-09-14T08:37:14.102 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:37:14.102 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T08:37:14.102 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:1 2024-09-14T08:37:14.102 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:37:14.102 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:37:14.102 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T08:37:14.103 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:1 2024-09-14T08:37:14.103 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:37:14.103 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:37:14.103 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:37:14.103 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:37:14.103 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T08:37:14.103 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:37:14.103 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:37:14.103 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=1 2024-09-14T08:37:14.103 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=1 2024-09-14T08:37:14.103 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T08:37:14.120 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:37:14.120 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:37:14.120 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:37:14.120 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2024-09-14T08:37:14.193 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T08:37:14.193 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:37:14.193 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:37:14.196 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:37:14.196 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2024-09-14T08:37:14.196 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:2 2024-09-14T08:37:14.196 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:37:14.196 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:37:14.196 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-14T08:37:14.196 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:2 2024-09-14T08:37:14.196 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:37:14.196 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:37:14.196 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:37:14.196 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=2 2024-09-14T08:37:14.196 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T08:37:14.197 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:37:14.197 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:37:14.197 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=2 2024-09-14T08:37:14.197 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:37:14.197 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-14T08:37:14.197 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:2 2024-09-14T08:37:14.197 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:37:14.197 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:37:14.197 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T08:37:14.197 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:2 2024-09-14T08:37:14.197 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:37:14.197 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:37:14.197 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:37:14.197 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:37:14.198 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-14T08:37:14.198 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:37:14.198 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:37:14.198 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=2 2024-09-14T08:37:14.198 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=2 2024-09-14T08:37:14.198 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T08:37:14.215 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:37:14.215 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:37:14.215 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:37:14.215 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2024-09-14T08:37:14.294 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ return 0 2024-09-14T08:37:14.295 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T08:37:14.295 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ echo ' "state": "Replaying"' 2024-09-14T08:37:14.295 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep -i Replaying 2024-09-14T08:37:14.296 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: "state": "Replaying" 2024-09-14T08:37:14.297 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_state=started 2024-09-14T08:37:14.297 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test started = started 2024-09-14T08:37:14.297 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:37:14.297 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2024-09-14T08:37:14.297 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:37:14.298 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:37:14.298 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:37:14.298 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:37:14.298 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = journal ']' 2024-09-14T08:37:14.298 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = snapshot ']' 2024-09-14T08:37:14.298 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test 2024-09-14T08:37:14.298 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:37:14.298 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:37:14.298 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:37:14.298 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:37:14.298 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster2-mirror-test.status 2024-09-14T08:37:14.298 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster2-mirror-test.status 2024-09-14T08:37:14.298 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:37:14.300 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-test.status 2024-09-14T08:37:14.300 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster1-mirror-test.status 2024-09-14T08:37:14.300 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster1-mirror-test.status 2024-09-14T08:37:14.300 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:37:14.300 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_status_log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:37:14.300 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ mirror_image_snapshot cluster2 mirror test 2024-09-14T08:37:14.300 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:37:14.300 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:37:14.300 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:37:14.300 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2024-09-14T08:37:15.990 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Snapshot ID: 8 2024-09-14T08:37:16.003 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.status 2024-09-14T08:37:16.003 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:37:16.003 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:37:16.003 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:37:16.003 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-test.status 2024-09-14T08:37:16.003 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2024-09-14T08:37:16.003 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:37:16.059 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-14T08:37:16.060 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snapshot_id=8 2024-09-14T08:37:16.060 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:37:16.060 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:37:16.060 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.2 2024-09-14T08:37:16.261 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:37:16.262 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:37:16.262 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:37:16.262 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:37:16.262 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:37:16.262 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-09-14T08:37:16.262 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:37:16.312 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:37:16.313 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:37:16.314 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:37:16.314 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:37:16.314 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:37:16.315 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:37:16.315 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 8 = '' 2024-09-14T08:37:16.315 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:37:16.315 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.4 2024-09-14T08:37:16.716 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:37:16.716 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:37:16.716 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:37:16.716 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:37:16.717 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:37:16.717 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-09-14T08:37:16.717 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:37:16.771 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:37:16.772 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:37:16.773 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:37:16.773 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:37:16.773 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:37:16.774 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:37:16.774 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 8 = '' 2024-09-14T08:37:16.774 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:37:16.774 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.8 2024-09-14T08:37:17.575 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:37:17.575 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:37:17.575 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:37:17.575 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:37:17.575 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:37:17.575 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-09-14T08:37:17.575 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:37:17.679 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:37:17.679 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:37:17.681 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:37:17.681 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:37:17.681 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:37:17.681 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:37:17.681 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 8 = '' 2024-09-14T08:37:17.681 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:37:17.682 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1.6 2024-09-14T08:37:19.283 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:37:19.283 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:37:19.283 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:37:19.283 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:37:19.283 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:37:19.283 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-09-14T08:37:19.283 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:37:19.335 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:37:19.337 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=8 2024-09-14T08:37:19.337 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 8 = 8 2024-09-14T08:37:19.337 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:37:19.338 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snap_present cluster1 mirror test snap2 2024-09-14T08:37:19.338 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:37:19.338 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:37:19.338 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:37:19.338 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap2 2024-09-14T08:37:19.338 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:37:19.338 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T08:37:19.338 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:37:20.338 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 info mirror/test@snap2 2024-09-14T08:37:20.388 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'test': 2024-09-14T08:37:20.389 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 512 MiB in 128 objects 2024-09-14T08:37:20.389 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:37:20.389 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 4 2024-09-14T08:37:20.389 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 10ba6324d73b 2024-09-14T08:37:20.389 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.10ba6324d73b 2024-09-14T08:37:20.389 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:37:20.389 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering, non-primary 2024-09-14T08:37:20.389 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:37:20.389 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:37:20.389 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:35:14 2024 2024-09-14T08:37:20.389 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:36:30 2024 2024-09-14T08:37:20.389 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:35:14 2024 2024-09-14T08:37:20.389 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: protected: False 2024-09-14T08:37:20.390 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:37:20.390 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:37:20.390 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: 1159fdda-fb30-49ff-ba1c-60890cdd16a0 2024-09-14T08:37:20.390 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: false 2024-09-14T08:37:20.392 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:37:20.392 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n snap1-clean ']' 2024-09-14T08:37:20.393 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ compare_image_snaps mirror test snap1-clean 2024-09-14T08:37:20.393 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:37:20.393 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:37:20.393 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap1-clean 2024-09-14T08:37:20.393 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local ret=0 2024-09-14T08:37:20.393 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local rmt_export=/tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export 2024-09-14T08:37:20.393 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local loc_export=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:37:20.393 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:37:20.393 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror export test@snap1-clean /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export 2024-09-14T08:37:21.026 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T08:37:21.037 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 -p mirror export test@snap1-clean /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:37:21.768 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T08:37:21.776 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cmp /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:37:21.989 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:37:22.106 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:37:22.107 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ compare_image_snaps mirror test snap2 2024-09-14T08:37:22.107 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:37:22.107 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:37:22.107 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap2 2024-09-14T08:37:22.107 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local ret=0 2024-09-14T08:37:22.107 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local rmt_export=/tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export 2024-09-14T08:37:22.107 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local loc_export=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:37:22.107 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:37:22.108 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror export test@snap2 /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export 2024-09-14T08:37:22.740 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T08:37:22.750 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 -p mirror export test@snap2 /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:37:23.536 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T08:37:23.545 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cmp /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:37:23.754 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:37:23.872 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:37:23.872 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ clean_snap_name=snap2-clean 2024-09-14T08:37:23.872 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_snap cluster2 mirror test snap2-clean 2024-09-14T08:37:23.872 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:37:23.873 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:37:23.873 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:37:23.873 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap2-clean 2024-09-14T08:37:23.873 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap2-clean --debug-rbd=20 --debug-journaler=20 2024-09-14T08:37:24.430 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 10` 2024-09-14T08:37:24.430 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ stress_write_image cluster2 mirror test 2024-09-14T08:37:24.430 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:37:24.430 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:37:24.431 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:37:24.431 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T08:37:24.432 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local duration=12 2024-09-14T08:37:24.432 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set +e 2024-09-14T08:37:24.432 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ timeout 12s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2024-09-14T08:37:24.672 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:100 IOs, 33985536 bytes 2024-09-14T08:37:24.747 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:200 IOs, 34597888 bytes 2024-09-14T08:37:24.864 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:300 IOs, 62425088 bytes 2024-09-14T08:37:25.103 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:400 IOs, 160359424 bytes 2024-09-14T08:37:25.245 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:500 IOs, 204395520 bytes 2024-09-14T08:37:25.409 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:600 IOs, 229497856 bytes 2024-09-14T08:37:25.563 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:700 IOs, 271035392 bytes 2024-09-14T08:37:25.661 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:800 IOs, 295492608 bytes 2024-09-14T08:37:25.849 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:900 IOs, 352723968 bytes 2024-09-14T08:37:25.962 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1000 IOs, 357978112 bytes 2024-09-14T08:37:26.053 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1100 IOs, 387709952 bytes 2024-09-14T08:37:26.249 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1200 IOs, 456610816 bytes 2024-09-14T08:37:26.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1300 IOs, 466092032 bytes 2024-09-14T08:37:26.516 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1400 IOs, 498735104 bytes 2024-09-14T08:37:26.604 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1500 IOs, 515784704 bytes 2024-09-14T08:37:26.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1600 IOs, 542426112 bytes 2024-09-14T08:37:26.840 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1700 IOs, 571682816 bytes 2024-09-14T08:37:26.979 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1800 IOs, 607771648 bytes 2024-09-14T08:37:27.143 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1900 IOs, 651098112 bytes 2024-09-14T08:37:27.258 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2000 IOs, 686300160 bytes 2024-09-14T08:37:27.475 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2100 IOs, 746985472 bytes 2024-09-14T08:37:27.615 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2200 IOs, 792241152 bytes 2024-09-14T08:37:27.680 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2300 IOs, 794956800 bytes 2024-09-14T08:37:27.853 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2400 IOs, 846419968 bytes 2024-09-14T08:37:28.009 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2500 IOs, 908432384 bytes 2024-09-14T08:37:28.211 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2600 IOs, 955082752 bytes 2024-09-14T08:37:28.339 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2700 IOs, 985701376 bytes 2024-09-14T08:37:28.496 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2800 IOs, 1029604352 bytes 2024-09-14T08:37:28.565 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2900 IOs, 1041876992 bytes 2024-09-14T08:37:28.747 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3000 IOs, 1095804928 bytes 2024-09-14T08:37:28.887 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3100 IOs, 1135800320 bytes 2024-09-14T08:37:28.971 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3200 IOs, 1150340096 bytes 2024-09-14T08:37:29.166 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3300 IOs, 1212416000 bytes 2024-09-14T08:37:29.416 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3400 IOs, 1302767616 bytes 2024-09-14T08:37:29.680 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3500 IOs, 1394001920 bytes 2024-09-14T08:37:29.761 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3600 IOs, 1402212352 bytes 2024-09-14T08:37:29.884 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3700 IOs, 1428062208 bytes 2024-09-14T08:37:30.036 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3800 IOs, 1503737856 bytes 2024-09-14T08:37:30.231 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3900 IOs, 1543811072 bytes 2024-09-14T08:37:30.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4000 IOs, 1568683008 bytes 2024-09-14T08:37:30.445 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4100 IOs, 1581815808 bytes 2024-09-14T08:37:30.646 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4200 IOs, 1651604480 bytes 2024-09-14T08:37:30.831 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4300 IOs, 1703006208 bytes 2024-09-14T08:37:30.961 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4400 IOs, 1741941760 bytes 2024-09-14T08:37:31.108 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4500 IOs, 1777388544 bytes 2024-09-14T08:37:31.287 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4600 IOs, 1830711296 bytes 2024-09-14T08:37:31.481 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4700 IOs, 1891682304 bytes 2024-09-14T08:37:31.639 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4800 IOs, 1934393344 bytes 2024-09-14T08:37:31.724 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4900 IOs, 1945494528 bytes 2024-09-14T08:37:31.794 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5000 IOs, 1955237888 bytes 2024-09-14T08:37:32.013 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5100 IOs, 2028367872 bytes 2024-09-14T08:37:32.153 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5200 IOs, 2065139712 bytes 2024-09-14T08:37:32.296 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5300 IOs, 2099013632 bytes 2024-09-14T08:37:32.353 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5400 IOs, 2099633152 bytes 2024-09-14T08:37:32.424 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5500 IOs, 2114495488 bytes 2024-09-14T08:37:32.512 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5600 IOs, 2115105792 bytes 2024-09-14T08:37:32.614 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5700 IOs, 2139753472 bytes 2024-09-14T08:37:32.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5800 IOs, 2157964288 bytes 2024-09-14T08:37:32.908 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5900 IOs, 2227196928 bytes 2024-09-14T08:37:33.068 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6000 IOs, 2261912576 bytes 2024-09-14T08:37:33.191 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6100 IOs, 2291115008 bytes 2024-09-14T08:37:33.244 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6200 IOs, 2307456000 bytes 2024-09-14T08:37:33.385 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6300 IOs, 2333796352 bytes 2024-09-14T08:37:33.510 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6400 IOs, 2361345024 bytes 2024-09-14T08:37:33.623 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6500 IOs, 2383489024 bytes 2024-09-14T08:37:33.807 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6600 IOs, 2440097792 bytes 2024-09-14T08:37:34.005 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6700 IOs, 2504238080 bytes 2024-09-14T08:37:34.117 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6800 IOs, 2530231296 bytes 2024-09-14T08:37:34.336 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6900 IOs, 2608673792 bytes 2024-09-14T08:37:34.572 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:7000 IOs, 2706636800 bytes 2024-09-14T08:37:34.844 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:7100 IOs, 2777244672 bytes 2024-09-14T08:37:34.924 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:7200 IOs, 2786456576 bytes 2024-09-14T08:37:35.114 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:7300 IOs, 2848525312 bytes 2024-09-14T08:37:35.307 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:7400 IOs, 2913769472 bytes 2024-09-14T08:37:35.477 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:7500 IOs, 2959676416 bytes 2024-09-14T08:37:35.551 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:7600 IOs, 2971567104 bytes 2024-09-14T08:37:35.631 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:7700 IOs, 2978428928 bytes 2024-09-14T08:37:35.966 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:7800 IOs, 3048806400 bytes 2024-09-14T08:37:36.075 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:7900 IOs, 3072681984 bytes 2024-09-14T08:37:36.218 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:8000 IOs, 3108431872 bytes 2024-09-14T08:37:36.319 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:8100 IOs, 3126139904 bytes 2024-09-14T08:37:36.423 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:8200 IOs, 3156126720 bytes 2024-09-14T08:37:36.437 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ error_code=124 2024-09-14T08:37:36.437 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set -e 2024-09-14T08:37:36.437 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T08:37:36.437 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:37:36.437 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2024-09-14T08:37:36.437 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:37:36.437 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:37:36.437 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:37:36.437 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state_pattern=up+replaying 2024-09-14T08:37:36.437 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local description_pattern= 2024-09-14T08:37:36.438 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local service_pattern= 2024-09-14T08:37:36.438 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:37:36.438 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:37:37.438 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2024-09-14T08:37:37.438 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:37:37.438 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:37:37.439 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:37:37.439 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state_pattern=up+replaying 2024-09-14T08:37:37.439 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local description_pattern= 2024-09-14T08:37:37.439 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local service_pattern= 2024-09-14T08:37:37.439 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status 2024-09-14T08:37:37.439 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ CEPH_ARGS= 2024-09-14T08:37:37.439 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2024-09-14T08:37:37.498 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ status='test: 2024-09-14T08:37:37.498 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: global_id: 1159fdda-fb30-49ff-ba1c-60890cdd16a0 2024-09-14T08:37:37.498 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: state: up+replaying 2024-09-14T08:37:37.498 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":526093312.0,"last_snapshot_bytes":526093312,"last_snapshot_sync_seconds":1,"local_snapshot_timestamp":1726302984,"remote_snapshot_timestamp":1726302984,"replay_state":"idle"} 2024-09-14T08:37:37.498 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: service: mirror.2 on smithi164 2024-09-14T08:37:37.498 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: last_update: 2024-09-14 08:37:12 2024-09-14T08:37:37.498 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: peer_sites: 2024-09-14T08:37:37.498 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: name: cluster2 2024-09-14T08:37:37.499 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: state: up+stopped 2024-09-14T08:37:37.499 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: description: local image is primary 2024-09-14T08:37:37.499 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: last_update: 2024-09-14 08:37:11' 2024-09-14T08:37:37.499 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep '^ state: .*up+replaying' 2024-09-14T08:37:37.499 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: state: up+replaying 2024-09-14T08:37:37.499 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep '^ description: .*' 2024-09-14T08:37:37.500 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":526093312.0,"last_snapshot_bytes":526093312,"last_snapshot_sync_seconds":1,"local_snapshot_timestamp":1726302984,"remote_snapshot_timestamp":1726302984,"replay_state":"idle"} 2024-09-14T08:37:37.500 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n '' ']' 2024-09-14T08:37:37.501 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ echo up+replaying 2024-09-14T08:37:37.501 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep '^up+' 2024-09-14T08:37:37.501 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:up+replaying 2024-09-14T08:37:37.501 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep 'service: *mirror..* on ' 2024-09-14T08:37:37.503 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: service: mirror.2 on smithi164 2024-09-14T08:37:37.503 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local last_update 2024-09-14T08:37:37.503 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-14T08:37:37.505 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ last_update='2024-09-14 08:37:12' 2024-09-14T08:37:37.505 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2024-09-14 08:37:12' 2024-09-14T08:37:37.506 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:37:37.506 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:37:37.506 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:37:37.506 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state_pattern=up+replaying 2024-09-14T08:37:37.506 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local 'prev_last_update=2024-09-14 08:37:12' 2024-09-14T08:37:37.506 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status 2024-09-14T08:37:37.506 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ CEPH_ARGS= 2024-09-14T08:37:37.506 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-14T08:37:37.565 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ status='OKOKOK142174290mirror.1smithi16419.3.0-4775-g7f24e589trueOK42214285mirror.2smithi16419.3.0-4775-g7f24e589falseOK43274330mirror.0smithi16419.3.0-4775-g7f24e589falseOK43604363mirror.3smithi16419.3.0-4775-g7f24e589falseOKtest1159fdda-fb30-49ff-ba1c-60890cdd16a0up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":526093312.0,"last_snapshot_bytes":526093312,"last_snapshot_sync_seconds":1,"local_snapshot_timestamp":1726302984,"remote_snapshot_timestamp":1726302984,"replay_state":"idle"}42214285mirror.2smithi1642024-09-14 08:37:12cluster29ab74346-7215-4f9b-943c-26e4f257da03up+stoppedlocal image is primary2024-09-14 08:37:11' 2024-09-14T08:37:37.565 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local last_update state 2024-09-14T08:37:37.565 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2024-09-14T08:37:37.566 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ last_update='2024-09-14 08:37:12' 2024-09-14T08:37:37.567 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2024-09-14T08:37:37.568 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ state=up+replaying 2024-09-14T08:37:37.569 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ echo up+replaying 2024-09-14T08:37:37.569 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep up+replaying 2024-09-14T08:37:37.569 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:up+replaying 2024-09-14T08:37:37.570 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:37:37.570 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:37:37.571 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ snap_name=snap3 2024-09-14T08:37:37.571 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_snap cluster2 mirror test snap3 2024-09-14T08:37:37.571 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:37:37.571 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:37:37.571 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:37:37.571 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap3 2024-09-14T08:37:37.571 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap3 --debug-rbd=20 --debug-journaler=20 2024-09-14T08:37:51.889 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.0 2024-09-14T08:37:51.890 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Sent signal 15 2024-09-14T08:37:51.890 INFO:tasks.rbd_mirror_thrash:waiting for 8 secs before reviving daemons 2024-09-14T08:37:51.891 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi164.stderr:2024-09-14T08:37:51.891+0000 7f3bc89ef640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 (PID: 37604) UID: 1000 2024-09-14T08:37:59.890 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.0 2024-09-14T08:37:59.891 INFO:teuthology.orchestra.run:waiting for 600 2024-09-14T08:37:59.891 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.0 2024-09-14T08:37:59.891 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Restarting daemon 2024-09-14T08:37:59.891 DEBUG:teuthology.orchestra.run.smithi164:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 2024-09-14T08:37:59.895 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Started 2024-09-14T08:37:59.895 INFO:tasks.rbd_mirror_thrash:waiting for 110 secs before thrashing 2024-09-14T08:38:04.946 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_started cluster1 mirror test 2024-09-14T08:38:04.946 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:38:04.946 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:38:04.946 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:38:04.946 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2024-09-14T08:38:04.946 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:38:04.946 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:38:04.946 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:38:04.946 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=started 2024-09-14T08:38:04.946 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:38:04.946 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:38:04.947 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:38:05.947 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_replay_state cluster1 mirror test started 2024-09-14T08:38:05.947 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:38:05.947 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:38:05.947 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:38:05.947 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=started 2024-09-14T08:38:05.947 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_result 2024-09-14T08:38:05.947 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=stopped 2024-09-14T08:38:05.948 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2024-09-14T08:38:05.948 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_instance=cluster1 2024-09-14T08:38:05.948 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:38:05.949 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ grep -i state 2024-09-14T08:38:05.949 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T08:38:05.949 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance=cluster1 2024-09-14T08:38:05.949 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local loop_instance 2024-09-14T08:38:05.949 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:38:05.949 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sleep 0 2024-09-14T08:38:05.949 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T08:38:05.951 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ seq 0 3 2024-09-14T08:38:05.951 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:38:05.951 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2024-09-14T08:38:05.951 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:0 2024-09-14T08:38:05.951 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:38:05.952 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:38:05.952 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T08:38:05.952 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:0 2024-09-14T08:38:05.952 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:38:05.952 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:38:05.952 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:38:05.952 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=0 2024-09-14T08:38:05.952 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T08:38:05.952 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:38:05.952 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:38:05.952 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=0 2024-09-14T08:38:05.952 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:38:05.952 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T08:38:05.952 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:0 2024-09-14T08:38:05.953 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:38:05.953 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:38:05.953 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T08:38:05.953 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:0 2024-09-14T08:38:05.953 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:38:05.953 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:38:05.953 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:38:05.953 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:38:05.953 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T08:38:05.953 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:38:05.953 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:38:05.953 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=0 2024-09-14T08:38:05.953 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=0 2024-09-14T08:38:05.953 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T08:38:05.975 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:38:05.975 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:38:05.975 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:38:05.975 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2024-09-14T08:38:06.056 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T08:38:06.056 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:38:06.056 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:38:06.059 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:38:06.059 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2024-09-14T08:38:06.059 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:1 2024-09-14T08:38:06.059 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:38:06.059 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:38:06.059 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T08:38:06.059 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:1 2024-09-14T08:38:06.060 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:38:06.060 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:38:06.060 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:38:06.060 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=1 2024-09-14T08:38:06.060 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T08:38:06.060 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:38:06.060 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:38:06.060 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=1 2024-09-14T08:38:06.060 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:38:06.060 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T08:38:06.060 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:1 2024-09-14T08:38:06.060 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:38:06.060 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:38:06.060 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T08:38:06.060 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:1 2024-09-14T08:38:06.061 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:38:06.061 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:38:06.061 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:38:06.061 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:38:06.061 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T08:38:06.061 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:38:06.061 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:38:06.061 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=1 2024-09-14T08:38:06.061 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=1 2024-09-14T08:38:06.061 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T08:38:06.079 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:38:06.079 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:38:06.079 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:38:06.079 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2024-09-14T08:38:06.152 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T08:38:06.152 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:38:06.152 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:38:06.155 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:38:06.155 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2024-09-14T08:38:06.155 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:2 2024-09-14T08:38:06.155 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:38:06.155 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:38:06.156 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-14T08:38:06.156 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:2 2024-09-14T08:38:06.156 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:38:06.156 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:38:06.156 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:38:06.156 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=2 2024-09-14T08:38:06.156 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T08:38:06.156 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:38:06.156 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:38:06.156 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=2 2024-09-14T08:38:06.156 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:38:06.156 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-14T08:38:06.156 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:2 2024-09-14T08:38:06.156 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:38:06.157 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:38:06.157 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T08:38:06.157 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:2 2024-09-14T08:38:06.157 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:38:06.157 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:38:06.157 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:38:06.157 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:38:06.157 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-14T08:38:06.157 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:38:06.157 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:38:06.157 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=2 2024-09-14T08:38:06.157 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=2 2024-09-14T08:38:06.157 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T08:38:06.174 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:38:06.174 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:38:06.174 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:38:06.175 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2024-09-14T08:38:06.255 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ return 0 2024-09-14T08:38:06.255 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T08:38:06.256 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ echo ' "state": "Replaying"' 2024-09-14T08:38:06.256 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep -i Replaying 2024-09-14T08:38:06.256 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: "state": "Replaying" 2024-09-14T08:38:06.258 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_state=started 2024-09-14T08:38:06.258 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test started = started 2024-09-14T08:38:06.258 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:38:06.258 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2024-09-14T08:38:06.258 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:38:06.258 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:38:06.258 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:38:06.258 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:38:06.258 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = journal ']' 2024-09-14T08:38:06.258 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = snapshot ']' 2024-09-14T08:38:06.258 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test 2024-09-14T08:38:06.259 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:38:06.259 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:38:06.259 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:38:06.259 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:38:06.259 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster2-mirror-test.status 2024-09-14T08:38:06.259 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster2-mirror-test.status 2024-09-14T08:38:06.259 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:38:06.260 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-test.status 2024-09-14T08:38:06.260 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster1-mirror-test.status 2024-09-14T08:38:06.260 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster1-mirror-test.status 2024-09-14T08:38:06.260 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:38:06.261 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_status_log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:38:06.261 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ mirror_image_snapshot cluster2 mirror test 2024-09-14T08:38:06.261 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:38:06.261 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:38:06.261 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:38:06.261 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2024-09-14T08:38:07.898 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.0 2024-09-14T08:38:07.899 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Sent signal 15 2024-09-14T08:38:07.899 INFO:tasks.rbd_mirror_thrash:waiting for 0 secs before reviving daemons 2024-09-14T08:38:07.899 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.0 2024-09-14T08:38:07.899 INFO:teuthology.orchestra.run:waiting for 600 2024-09-14T08:38:07.900 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi164.stderr:2024-09-14T08:38:07.900+0000 7f8050e0b640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 (PID: 38803) UID: 1000 2024-09-14T08:38:07.950 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Snapshot ID: 11 2024-09-14T08:38:07.959 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.status 2024-09-14T08:38:07.960 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:38:07.960 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:38:07.960 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:38:07.960 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-test.status 2024-09-14T08:38:07.960 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:38:07.960 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2024-09-14T08:38:08.030 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-14T08:38:08.032 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snapshot_id=11 2024-09-14T08:38:08.032 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:38:08.032 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:38:08.032 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.2 2024-09-14T08:38:08.233 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:38:08.233 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:38:08.233 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:38:08.233 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:38:08.234 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:38:08.234 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-09-14T08:38:08.234 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:38:08.285 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:38:08.285 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:38:08.287 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:38:08.287 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:38:08.287 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:38:08.287 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:38:08.287 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 11 = '' 2024-09-14T08:38:08.287 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:38:08.287 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.4 2024-09-14T08:38:08.688 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:38:08.688 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:38:08.688 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:38:08.689 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:38:08.689 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:38:08.689 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-09-14T08:38:08.689 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:38:08.742 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:38:08.742 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:38:08.744 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:38:08.744 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:38:08.744 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:38:08.744 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:38:08.744 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 11 = '' 2024-09-14T08:38:08.744 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:38:08.745 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.8 2024-09-14T08:38:09.546 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:38:09.546 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:38:09.546 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:38:09.546 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:38:09.546 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:38:09.546 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-09-14T08:38:09.546 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:38:09.598 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:38:09.598 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:38:09.599 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:38:09.599 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:38:09.599 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:38:09.600 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:38:09.600 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 11 = '' 2024-09-14T08:38:09.600 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:38:09.600 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1.6 2024-09-14T08:38:11.201 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:38:11.202 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:38:11.202 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:38:11.202 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:38:11.202 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:38:11.202 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-09-14T08:38:11.202 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:38:11.249 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:38:11.251 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=11 2024-09-14T08:38:11.251 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 11 = 11 2024-09-14T08:38:11.251 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:38:11.251 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snap_present cluster1 mirror test snap3 2024-09-14T08:38:11.252 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:38:11.252 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:38:11.252 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:38:11.252 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap3 2024-09-14T08:38:11.252 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:38:11.252 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T08:38:11.252 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:38:12.252 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 info mirror/test@snap3 2024-09-14T08:38:12.299 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'test': 2024-09-14T08:38:12.299 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 512 MiB in 128 objects 2024-09-14T08:38:12.299 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:38:12.300 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 6 2024-09-14T08:38:12.300 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 10ba6324d73b 2024-09-14T08:38:12.300 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.10ba6324d73b 2024-09-14T08:38:12.300 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:38:12.300 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering, non-primary 2024-09-14T08:38:12.300 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:38:12.300 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:38:12.300 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:35:14 2024 2024-09-14T08:38:12.300 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:36:30 2024 2024-09-14T08:38:12.300 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:35:14 2024 2024-09-14T08:38:12.300 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: protected: False 2024-09-14T08:38:12.300 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:38:12.300 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:38:12.301 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: 1159fdda-fb30-49ff-ba1c-60890cdd16a0 2024-09-14T08:38:12.301 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: false 2024-09-14T08:38:12.303 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:38:12.304 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n snap2-clean ']' 2024-09-14T08:38:12.304 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ compare_image_snaps mirror test snap2-clean 2024-09-14T08:38:12.304 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:38:12.304 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:38:12.304 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap2-clean 2024-09-14T08:38:12.304 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local ret=0 2024-09-14T08:38:12.304 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local rmt_export=/tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export 2024-09-14T08:38:12.304 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local loc_export=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:38:12.304 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:38:12.305 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror export test@snap2-clean /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export 2024-09-14T08:38:12.933 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T08:38:12.943 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 -p mirror export test@snap2-clean /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:38:13.629 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T08:38:13.638 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cmp /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:38:13.849 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:38:13.968 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:38:13.968 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ compare_image_snaps mirror test snap3 2024-09-14T08:38:13.968 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:38:13.968 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:38:13.968 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap3 2024-09-14T08:38:13.968 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local ret=0 2024-09-14T08:38:13.968 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local rmt_export=/tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export 2024-09-14T08:38:13.969 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local loc_export=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:38:13.969 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:38:13.969 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror export test@snap3 /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export 2024-09-14T08:38:14.001 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.0 2024-09-14T08:38:14.001 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Restarting daemon 2024-09-14T08:38:14.001 DEBUG:teuthology.orchestra.run.smithi164:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 2024-09-14T08:38:14.003 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Started 2024-09-14T08:38:14.004 INFO:tasks.rbd_mirror_thrash:waiting for 111 secs before thrashing 2024-09-14T08:38:14.627 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T08:38:14.637 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 -p mirror export test@snap3 /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:38:15.484 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T08:38:15.494 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cmp /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:38:15.704 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:38:15.822 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:38:15.822 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ clean_snap_name=snap3-clean 2024-09-14T08:38:15.822 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_snap cluster2 mirror test snap3-clean 2024-09-14T08:38:15.822 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:38:15.822 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:38:15.822 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:38:15.822 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap3-clean 2024-09-14T08:38:15.822 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap3-clean --debug-rbd=20 --debug-journaler=20 2024-09-14T08:38:16.432 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 10` 2024-09-14T08:38:16.432 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ stress_write_image cluster2 mirror test 2024-09-14T08:38:16.433 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:38:16.433 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:38:16.433 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:38:16.433 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T08:38:16.434 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local duration=6 2024-09-14T08:38:16.434 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set +e 2024-09-14T08:38:16.434 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ timeout 6s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2024-09-14T08:38:16.639 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:100 IOs, 22266880 bytes 2024-09-14T08:38:16.834 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:200 IOs, 70804480 bytes 2024-09-14T08:38:16.982 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:300 IOs, 120196096 bytes 2024-09-14T08:38:17.102 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:400 IOs, 126865408 bytes 2024-09-14T08:38:17.360 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:500 IOs, 208600064 bytes 2024-09-14T08:38:17.525 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:600 IOs, 252747776 bytes 2024-09-14T08:38:17.684 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:700 IOs, 298952704 bytes 2024-09-14T08:38:17.842 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:800 IOs, 335619072 bytes 2024-09-14T08:38:17.959 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:900 IOs, 359468032 bytes 2024-09-14T08:38:18.132 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1000 IOs, 410877952 bytes 2024-09-14T08:38:18.257 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1100 IOs, 437776384 bytes 2024-09-14T08:38:18.386 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1200 IOs, 469242880 bytes 2024-09-14T08:38:18.557 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1300 IOs, 513980416 bytes 2024-09-14T08:38:18.704 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1400 IOs, 561799168 bytes 2024-09-14T08:38:18.875 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1500 IOs, 616875008 bytes 2024-09-14T08:38:19.071 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1600 IOs, 681902080 bytes 2024-09-14T08:38:19.220 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1700 IOs, 702173184 bytes 2024-09-14T08:38:19.316 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1800 IOs, 711299072 bytes 2024-09-14T08:38:19.477 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1900 IOs, 757074944 bytes 2024-09-14T08:38:19.651 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2000 IOs, 808379392 bytes 2024-09-14T08:38:19.727 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2100 IOs, 826181632 bytes 2024-09-14T08:38:19.871 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2200 IOs, 853231616 bytes 2024-09-14T08:38:19.994 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2300 IOs, 883701760 bytes 2024-09-14T08:38:20.126 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2400 IOs, 914105344 bytes 2024-09-14T08:38:20.200 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2500 IOs, 920184832 bytes 2024-09-14T08:38:20.307 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2600 IOs, 945354752 bytes 2024-09-14T08:38:20.449 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2700 IOs, 987944960 bytes 2024-09-14T08:38:20.533 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2800 IOs, 997480448 bytes 2024-09-14T08:38:20.610 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2900 IOs, 1012395008 bytes 2024-09-14T08:38:20.824 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3000 IOs, 1075244032 bytes 2024-09-14T08:38:20.962 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3100 IOs, 1119511552 bytes 2024-09-14T08:38:21.119 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3200 IOs, 1161656320 bytes 2024-09-14T08:38:21.269 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3300 IOs, 1214968832 bytes 2024-09-14T08:38:21.475 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3400 IOs, 1277139968 bytes 2024-09-14T08:38:21.701 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3500 IOs, 1333224448 bytes 2024-09-14T08:38:21.819 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3600 IOs, 1377820672 bytes 2024-09-14T08:38:22.020 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3700 IOs, 1421410304 bytes 2024-09-14T08:38:22.197 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3800 IOs, 1469257728 bytes 2024-09-14T08:38:22.358 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3900 IOs, 1539345408 bytes 2024-09-14T08:38:22.441 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ error_code=124 2024-09-14T08:38:22.442 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set -e 2024-09-14T08:38:22.442 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T08:38:22.442 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:38:22.442 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2024-09-14T08:38:22.442 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:38:22.442 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:38:22.442 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:38:22.442 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state_pattern=up+replaying 2024-09-14T08:38:22.442 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local description_pattern= 2024-09-14T08:38:22.442 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local service_pattern= 2024-09-14T08:38:22.442 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:38:22.442 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:38:23.442 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2024-09-14T08:38:23.443 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:38:23.443 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:38:23.443 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:38:23.443 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state_pattern=up+replaying 2024-09-14T08:38:23.443 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local description_pattern= 2024-09-14T08:38:23.443 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local service_pattern= 2024-09-14T08:38:23.443 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status 2024-09-14T08:38:23.443 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ CEPH_ARGS= 2024-09-14T08:38:23.443 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2024-09-14T08:38:23.500 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ status='test: 2024-09-14T08:38:23.501 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: global_id: 1159fdda-fb30-49ff-ba1c-60890cdd16a0 2024-09-14T08:38:23.501 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: state: up+replaying 2024-09-14T08:38:23.501 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: description: replaying, {"bytes_per_second":17853303.47,"bytes_per_snapshot":533881856.0,"last_snapshot_bytes":535599104,"last_snapshot_sync_seconds":1,"local_snapshot_timestamp":1726303086,"remote_snapshot_timestamp":1726303086,"replay_state":"idle"} 2024-09-14T08:38:23.501 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: service: mirror.2 on smithi164 2024-09-14T08:38:23.501 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: last_update: 2024-09-14 08:38:12 2024-09-14T08:38:23.501 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: peer_sites: 2024-09-14T08:38:23.501 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: name: cluster2 2024-09-14T08:38:23.501 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: state: up+stopped 2024-09-14T08:38:23.501 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: description: local image is primary 2024-09-14T08:38:23.501 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: last_update: 2024-09-14 08:38:11' 2024-09-14T08:38:23.501 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep '^ state: .*up+replaying' 2024-09-14T08:38:23.502 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: state: up+replaying 2024-09-14T08:38:23.502 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep '^ description: .*' 2024-09-14T08:38:23.502 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: description: replaying, {"bytes_per_second":17853303.47,"bytes_per_snapshot":533881856.0,"last_snapshot_bytes":535599104,"last_snapshot_sync_seconds":1,"local_snapshot_timestamp":1726303086,"remote_snapshot_timestamp":1726303086,"replay_state":"idle"} 2024-09-14T08:38:23.503 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n '' ']' 2024-09-14T08:38:23.503 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ echo up+replaying 2024-09-14T08:38:23.503 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep '^up+' 2024-09-14T08:38:23.504 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:up+replaying 2024-09-14T08:38:23.504 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep 'service: *mirror..* on ' 2024-09-14T08:38:23.505 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: service: mirror.2 on smithi164 2024-09-14T08:38:23.505 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local last_update 2024-09-14T08:38:23.506 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-14T08:38:23.507 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ last_update='2024-09-14 08:38:12' 2024-09-14T08:38:23.507 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2024-09-14 08:38:12' 2024-09-14T08:38:23.508 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:38:23.508 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:38:23.508 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:38:23.508 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state_pattern=up+replaying 2024-09-14T08:38:23.508 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local 'prev_last_update=2024-09-14 08:38:12' 2024-09-14T08:38:23.508 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status 2024-09-14T08:38:23.508 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ CEPH_ARGS= 2024-09-14T08:38:23.508 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-14T08:38:23.568 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ status='OKOKOK142174290mirror.1smithi16419.3.0-4775-g7f24e589trueOK42214285mirror.2smithi16419.3.0-4775-g7f24e589falseOK43604363mirror.3smithi16419.3.0-4775-g7f24e589falseOK44194422mirror.0smithi16419.3.0-4775-g7f24e589falseOKtest1159fdda-fb30-49ff-ba1c-60890cdd16a0up+replayingreplaying, {"bytes_per_second":17853303.47,"bytes_per_snapshot":533881856.0,"last_snapshot_bytes":535599104,"last_snapshot_sync_seconds":1,"local_snapshot_timestamp":1726303086,"remote_snapshot_timestamp":1726303086,"replay_state":"idle"}42214285mirror.2smithi1642024-09-14 08:38:12cluster29ab74346-7215-4f9b-943c-26e4f257da03up+stoppedlocal image is primary2024-09-14 08:38:11' 2024-09-14T08:38:23.568 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local last_update state 2024-09-14T08:38:23.568 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2024-09-14T08:38:23.570 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ last_update='2024-09-14 08:38:12' 2024-09-14T08:38:23.570 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2024-09-14T08:38:23.571 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ state=up+replaying 2024-09-14T08:38:23.572 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ echo up+replaying 2024-09-14T08:38:23.572 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep up+replaying 2024-09-14T08:38:23.573 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:up+replaying 2024-09-14T08:38:23.573 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:38:23.573 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:38:23.574 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ snap_name=snap4 2024-09-14T08:38:23.574 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_snap cluster2 mirror test snap4 2024-09-14T08:38:23.574 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:38:23.574 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:38:23.574 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:38:23.574 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap4 2024-09-14T08:38:23.574 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap4 --debug-rbd=20 --debug-journaler=20 2024-09-14T08:38:52.480 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_started cluster1 mirror test 2024-09-14T08:38:52.480 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:38:52.480 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:38:52.480 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:38:52.480 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2024-09-14T08:38:52.480 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:38:52.481 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:38:52.481 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:38:52.481 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=started 2024-09-14T08:38:52.481 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:38:52.481 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:38:52.481 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:38:53.481 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_replay_state cluster1 mirror test started 2024-09-14T08:38:53.481 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:38:53.481 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:38:53.481 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:38:53.481 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=started 2024-09-14T08:38:53.482 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_result 2024-09-14T08:38:53.482 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=stopped 2024-09-14T08:38:53.483 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2024-09-14T08:38:53.483 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ grep -i state 2024-09-14T08:38:53.483 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_instance=cluster1 2024-09-14T08:38:53.483 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:38:53.483 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T08:38:53.483 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance=cluster1 2024-09-14T08:38:53.483 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local loop_instance 2024-09-14T08:38:53.483 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:38:53.483 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sleep 0 2024-09-14T08:38:53.483 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T08:38:53.485 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ seq 0 3 2024-09-14T08:38:53.485 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:38:53.485 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2024-09-14T08:38:53.485 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:0 2024-09-14T08:38:53.486 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:38:53.486 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:38:53.486 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T08:38:53.486 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:0 2024-09-14T08:38:53.486 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:38:53.486 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:38:53.486 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:38:53.486 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=0 2024-09-14T08:38:53.486 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T08:38:53.486 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:38:53.486 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:38:53.486 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=0 2024-09-14T08:38:53.486 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:38:53.486 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T08:38:53.487 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:0 2024-09-14T08:38:53.487 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:38:53.487 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:38:53.487 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T08:38:53.488 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:0 2024-09-14T08:38:53.488 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:38:53.488 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:38:53.488 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:38:53.488 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:38:53.488 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T08:38:53.488 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:38:53.489 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:38:53.489 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=0 2024-09-14T08:38:53.489 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=0 2024-09-14T08:38:53.489 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T08:38:53.508 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:38:53.508 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:38:53.508 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:38:53.508 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2024-09-14T08:38:53.587 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T08:38:53.587 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:38:53.587 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:38:53.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:38:53.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2024-09-14T08:38:53.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:1 2024-09-14T08:38:53.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:38:53.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:38:53.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T08:38:53.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:1 2024-09-14T08:38:53.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:38:53.591 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:38:53.591 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:38:53.591 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=1 2024-09-14T08:38:53.591 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T08:38:53.591 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:38:53.591 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:38:53.591 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=1 2024-09-14T08:38:53.591 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:38:53.591 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T08:38:53.591 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:1 2024-09-14T08:38:53.591 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:38:53.591 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:38:53.592 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T08:38:53.592 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:1 2024-09-14T08:38:53.592 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:38:53.592 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:38:53.592 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:38:53.592 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:38:53.592 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T08:38:53.592 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:38:53.592 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:38:53.592 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=1 2024-09-14T08:38:53.592 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=1 2024-09-14T08:38:53.592 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T08:38:53.609 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:38:53.609 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:38:53.609 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:38:53.609 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2024-09-14T08:38:53.681 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T08:38:53.681 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:38:53.681 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:38:53.685 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:38:53.685 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2024-09-14T08:38:53.685 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:2 2024-09-14T08:38:53.685 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:38:53.685 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:38:53.685 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-14T08:38:53.685 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:2 2024-09-14T08:38:53.685 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:38:53.685 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:38:53.685 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:38:53.685 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=2 2024-09-14T08:38:53.685 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T08:38:53.685 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:38:53.686 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:38:53.686 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=2 2024-09-14T08:38:53.686 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:38:53.686 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-14T08:38:53.686 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:2 2024-09-14T08:38:53.686 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:38:53.686 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:38:53.686 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T08:38:53.686 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:2 2024-09-14T08:38:53.686 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:38:53.686 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:38:53.686 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:38:53.686 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:38:53.687 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-14T08:38:53.687 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:38:53.687 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:38:53.687 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=2 2024-09-14T08:38:53.687 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=2 2024-09-14T08:38:53.687 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T08:38:53.703 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:38:53.703 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:38:53.703 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:38:53.703 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2024-09-14T08:38:53.783 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ return 0 2024-09-14T08:38:53.784 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T08:38:53.784 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ echo ' "state": "Replaying"' 2024-09-14T08:38:53.784 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep -i Replaying 2024-09-14T08:38:53.785 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: "state": "Replaying" 2024-09-14T08:38:53.787 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_state=started 2024-09-14T08:38:53.787 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test started = started 2024-09-14T08:38:53.787 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:38:53.787 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2024-09-14T08:38:53.787 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:38:53.787 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:38:53.787 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:38:53.787 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:38:53.787 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = journal ']' 2024-09-14T08:38:53.787 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = snapshot ']' 2024-09-14T08:38:53.787 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test 2024-09-14T08:38:53.787 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:38:53.788 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:38:53.788 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:38:53.788 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:38:53.788 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster2-mirror-test.status 2024-09-14T08:38:53.788 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster2-mirror-test.status 2024-09-14T08:38:53.788 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:38:53.789 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-test.status 2024-09-14T08:38:53.789 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster1-mirror-test.status 2024-09-14T08:38:53.789 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster1-mirror-test.status 2024-09-14T08:38:53.789 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:38:53.789 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_status_log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:38:53.790 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ mirror_image_snapshot cluster2 mirror test 2024-09-14T08:38:53.790 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:38:53.790 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:38:53.790 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:38:53.790 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2024-09-14T08:38:55.486 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Snapshot ID: 14 2024-09-14T08:38:55.499 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.status 2024-09-14T08:38:55.499 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:38:55.499 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:38:55.499 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:38:55.499 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-test.status 2024-09-14T08:38:55.499 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2024-09-14T08:38:55.500 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:38:55.566 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-14T08:38:55.568 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snapshot_id=14 2024-09-14T08:38:55.568 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:38:55.568 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:38:55.568 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.2 2024-09-14T08:38:55.770 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:38:55.770 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:38:55.770 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:38:55.770 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:38:55.770 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:38:55.770 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-09-14T08:38:55.770 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:38:55.821 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:38:55.821 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:38:55.824 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:38:55.825 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:38:55.825 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:38:55.825 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:38:55.825 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 14 = '' 2024-09-14T08:38:55.825 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:38:55.825 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.4 2024-09-14T08:38:56.224 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:38:56.224 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:38:56.224 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:38:56.224 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:38:56.224 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:38:56.224 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-09-14T08:38:56.224 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:38:56.274 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:38:56.274 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:38:56.276 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:38:56.276 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:38:56.276 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:38:56.276 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:38:56.276 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 14 = '' 2024-09-14T08:38:56.276 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:38:56.276 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.8 2024-09-14T08:38:57.077 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:38:57.077 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:38:57.078 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:38:57.078 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:38:57.078 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:38:57.078 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-09-14T08:38:57.078 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:38:57.133 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:38:57.133 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:38:57.134 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:38:57.135 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:38:57.135 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:38:57.135 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:38:57.135 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 14 = '' 2024-09-14T08:38:57.135 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:38:57.135 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1.6 2024-09-14T08:38:58.737 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:38:58.737 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:38:58.737 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:38:58.737 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:38:58.737 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:38:58.737 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-09-14T08:38:58.737 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:38:58.785 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:38:58.787 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=14 2024-09-14T08:38:58.787 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 14 = 14 2024-09-14T08:38:58.787 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:38:58.787 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snap_present cluster1 mirror test snap4 2024-09-14T08:38:58.787 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:38:58.787 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:38:58.787 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:38:58.787 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap4 2024-09-14T08:38:58.788 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:38:58.788 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T08:38:58.788 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:38:59.788 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 info mirror/test@snap4 2024-09-14T08:38:59.837 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'test': 2024-09-14T08:38:59.837 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 512 MiB in 128 objects 2024-09-14T08:38:59.837 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:38:59.837 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 8 2024-09-14T08:38:59.837 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 10ba6324d73b 2024-09-14T08:38:59.837 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.10ba6324d73b 2024-09-14T08:38:59.837 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:38:59.837 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering, non-primary 2024-09-14T08:38:59.837 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:38:59.837 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:38:59.837 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:35:14 2024 2024-09-14T08:38:59.838 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:38:12 2024 2024-09-14T08:38:59.838 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:35:14 2024 2024-09-14T08:38:59.838 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: protected: False 2024-09-14T08:38:59.838 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:38:59.838 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:38:59.838 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: 1159fdda-fb30-49ff-ba1c-60890cdd16a0 2024-09-14T08:38:59.838 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: false 2024-09-14T08:38:59.841 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:38:59.841 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n snap3-clean ']' 2024-09-14T08:38:59.841 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ compare_image_snaps mirror test snap3-clean 2024-09-14T08:38:59.841 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:38:59.841 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:38:59.841 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap3-clean 2024-09-14T08:38:59.841 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local ret=0 2024-09-14T08:38:59.841 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local rmt_export=/tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export 2024-09-14T08:38:59.841 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local loc_export=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:38:59.841 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:38:59.842 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror export test@snap3-clean /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export 2024-09-14T08:39:00.474 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T08:39:00.483 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 -p mirror export test@snap3-clean /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:39:01.219 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T08:39:01.226 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cmp /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:39:01.437 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:39:01.551 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:39:01.551 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ compare_image_snaps mirror test snap4 2024-09-14T08:39:01.551 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:39:01.551 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:39:01.552 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap4 2024-09-14T08:39:01.552 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local ret=0 2024-09-14T08:39:01.552 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local rmt_export=/tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export 2024-09-14T08:39:01.552 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local loc_export=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:39:01.552 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:39:01.552 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror export test@snap4 /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export 2024-09-14T08:39:02.174 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T08:39:02.185 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 -p mirror export test@snap4 /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:39:03.030 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T08:39:03.039 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cmp /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:39:03.250 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:39:03.365 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:39:03.365 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ clean_snap_name=snap4-clean 2024-09-14T08:39:03.365 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_snap cluster2 mirror test snap4-clean 2024-09-14T08:39:03.365 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:39:03.366 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:39:03.366 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:39:03.366 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap4-clean 2024-09-14T08:39:03.366 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap4-clean --debug-rbd=20 --debug-journaler=20 2024-09-14T08:39:03.436 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 10` 2024-09-14T08:39:03.436 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ stress_write_image cluster2 mirror test 2024-09-14T08:39:03.436 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:39:03.436 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:39:03.436 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:39:03.437 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T08:39:03.437 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local duration=12 2024-09-14T08:39:03.438 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set +e 2024-09-14T08:39:03.438 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ timeout 12s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2024-09-14T08:39:03.716 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:100 IOs, 55788544 bytes 2024-09-14T08:39:03.874 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:200 IOs, 84560896 bytes 2024-09-14T08:39:03.997 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:300 IOs, 108304384 bytes 2024-09-14T08:39:04.133 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:400 IOs, 144574464 bytes 2024-09-14T08:39:04.286 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:500 IOs, 189371392 bytes 2024-09-14T08:39:04.487 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:600 IOs, 262168576 bytes 2024-09-14T08:39:04.706 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:700 IOs, 316720128 bytes 2024-09-14T08:39:04.883 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:800 IOs, 369965056 bytes 2024-09-14T08:39:04.986 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:900 IOs, 392192000 bytes 2024-09-14T08:39:05.116 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1000 IOs, 426889216 bytes 2024-09-14T08:39:05.255 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1100 IOs, 461372416 bytes 2024-09-14T08:39:05.374 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1200 IOs, 485488640 bytes 2024-09-14T08:39:05.444 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1300 IOs, 515076096 bytes 2024-09-14T08:39:05.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1400 IOs, 528110592 bytes 2024-09-14T08:39:05.787 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1500 IOs, 598950912 bytes 2024-09-14T08:39:05.955 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1600 IOs, 637136896 bytes 2024-09-14T08:39:06.106 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1700 IOs, 690793472 bytes 2024-09-14T08:39:06.357 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1800 IOs, 760517632 bytes 2024-09-14T08:39:06.449 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1900 IOs, 774178816 bytes 2024-09-14T08:39:06.591 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2000 IOs, 814474240 bytes 2024-09-14T08:39:06.677 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2100 IOs, 823010304 bytes 2024-09-14T08:39:06.878 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2200 IOs, 885187584 bytes 2024-09-14T08:39:07.116 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2300 IOs, 956800000 bytes 2024-09-14T08:39:07.354 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2400 IOs, 1033771008 bytes 2024-09-14T08:39:07.511 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2500 IOs, 1080200192 bytes 2024-09-14T08:39:07.707 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2600 IOs, 1139253248 bytes 2024-09-14T08:39:07.820 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2700 IOs, 1162205184 bytes 2024-09-14T08:39:07.910 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2800 IOs, 1187976192 bytes 2024-09-14T08:39:08.108 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2900 IOs, 1240896512 bytes 2024-09-14T08:39:08.220 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3000 IOs, 1262726144 bytes 2024-09-14T08:39:08.442 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3100 IOs, 1330037760 bytes 2024-09-14T08:39:08.520 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3200 IOs, 1337392128 bytes 2024-09-14T08:39:08.736 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3300 IOs, 1412989952 bytes 2024-09-14T08:39:08.827 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3400 IOs, 1429660672 bytes 2024-09-14T08:39:09.065 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3500 IOs, 1499023360 bytes 2024-09-14T08:39:09.219 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3600 IOs, 1559095296 bytes 2024-09-14T08:39:09.331 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3700 IOs, 1560264704 bytes 2024-09-14T08:39:09.429 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3800 IOs, 1579983872 bytes 2024-09-14T08:39:09.554 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3900 IOs, 1610600448 bytes 2024-09-14T08:39:09.650 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4000 IOs, 1632206848 bytes 2024-09-14T08:39:09.804 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4100 IOs, 1678713856 bytes 2024-09-14T08:39:09.896 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4200 IOs, 1684820992 bytes 2024-09-14T08:39:09.990 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4300 IOs, 1718233088 bytes 2024-09-14T08:39:10.258 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4400 IOs, 1793612800 bytes 2024-09-14T08:39:10.433 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4500 IOs, 1834230784 bytes 2024-09-14T08:39:10.549 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4600 IOs, 1856404480 bytes 2024-09-14T08:39:10.667 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4700 IOs, 1883613184 bytes 2024-09-14T08:39:10.730 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4800 IOs, 1885299712 bytes 2024-09-14T08:39:10.821 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4900 IOs, 1919091712 bytes 2024-09-14T08:39:10.986 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5000 IOs, 1960322048 bytes 2024-09-14T08:39:11.218 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5100 IOs, 2021864448 bytes 2024-09-14T08:39:11.393 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5200 IOs, 2081136640 bytes 2024-09-14T08:39:11.557 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5300 IOs, 2127357952 bytes 2024-09-14T08:39:11.779 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5400 IOs, 2196080640 bytes 2024-09-14T08:39:11.991 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5500 IOs, 2273660928 bytes 2024-09-14T08:39:12.126 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5600 IOs, 2308315136 bytes 2024-09-14T08:39:12.391 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5700 IOs, 2399060992 bytes 2024-09-14T08:39:12.747 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5800 IOs, 2512813056 bytes 2024-09-14T08:39:13.020 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5900 IOs, 2599678976 bytes 2024-09-14T08:39:13.167 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6000 IOs, 2640502784 bytes 2024-09-14T08:39:13.361 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6100 IOs, 2711896064 bytes 2024-09-14T08:39:13.653 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6200 IOs, 2807602176 bytes 2024-09-14T08:39:13.729 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6300 IOs, 2817644544 bytes 2024-09-14T08:39:13.827 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6400 IOs, 2844778496 bytes 2024-09-14T08:39:14.076 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6500 IOs, 2930329600 bytes 2024-09-14T08:39:14.320 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6600 IOs, 3001755648 bytes 2024-09-14T08:39:14.430 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6700 IOs, 3023471616 bytes 2024-09-14T08:39:14.596 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6800 IOs, 3070220288 bytes 2024-09-14T08:39:14.718 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6900 IOs, 3105650688 bytes 2024-09-14T08:39:14.915 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:7000 IOs, 3162161152 bytes 2024-09-14T08:39:15.096 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:7100 IOs, 3212372992 bytes 2024-09-14T08:39:15.281 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:7200 IOs, 3272931328 bytes 2024-09-14T08:39:15.445 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ error_code=124 2024-09-14T08:39:15.445 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set -e 2024-09-14T08:39:15.445 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T08:39:15.445 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:39:15.445 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2024-09-14T08:39:15.445 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:39:15.445 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:39:15.445 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:39:15.446 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state_pattern=up+replaying 2024-09-14T08:39:15.446 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local description_pattern= 2024-09-14T08:39:15.446 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local service_pattern= 2024-09-14T08:39:15.446 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:39:15.446 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:39:16.447 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2024-09-14T08:39:16.447 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:39:16.447 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:39:16.447 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:39:16.447 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state_pattern=up+replaying 2024-09-14T08:39:16.447 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local description_pattern= 2024-09-14T08:39:16.447 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local service_pattern= 2024-09-14T08:39:16.447 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status 2024-09-14T08:39:16.447 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ CEPH_ARGS= 2024-09-14T08:39:16.447 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2024-09-14T08:39:16.506 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ status='test: 2024-09-14T08:39:16.506 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: global_id: 1159fdda-fb30-49ff-ba1c-60890cdd16a0 2024-09-14T08:39:16.506 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: state: up+replaying 2024-09-14T08:39:16.507 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: description: replaying, {"bytes_per_second":16922419.2,"bytes_per_snapshot":521635840.0,"last_snapshot_bytes":507672576,"last_snapshot_sync_seconds":1,"local_snapshot_timestamp":1726303134,"remote_snapshot_timestamp":1726303134,"replay_state":"idle"} 2024-09-14T08:39:16.507 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: service: mirror.2 on smithi164 2024-09-14T08:39:16.507 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: last_update: 2024-09-14 08:39:12 2024-09-14T08:39:16.507 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: peer_sites: 2024-09-14T08:39:16.507 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: name: cluster2 2024-09-14T08:39:16.507 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: state: up+stopped 2024-09-14T08:39:16.507 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: description: local image is primary 2024-09-14T08:39:16.507 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: last_update: 2024-09-14 08:39:11' 2024-09-14T08:39:16.507 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep '^ state: .*up+replaying' 2024-09-14T08:39:16.508 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: state: up+replaying 2024-09-14T08:39:16.508 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep '^ description: .*' 2024-09-14T08:39:16.508 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: description: replaying, {"bytes_per_second":16922419.2,"bytes_per_snapshot":521635840.0,"last_snapshot_bytes":507672576,"last_snapshot_sync_seconds":1,"local_snapshot_timestamp":1726303134,"remote_snapshot_timestamp":1726303134,"replay_state":"idle"} 2024-09-14T08:39:16.509 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n '' ']' 2024-09-14T08:39:16.509 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ echo up+replaying 2024-09-14T08:39:16.509 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep '^up+' 2024-09-14T08:39:16.510 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:up+replaying 2024-09-14T08:39:16.510 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep 'service: *mirror..* on ' 2024-09-14T08:39:16.511 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: service: mirror.2 on smithi164 2024-09-14T08:39:16.511 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local last_update 2024-09-14T08:39:16.511 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-14T08:39:16.513 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ last_update='2024-09-14 08:39:12' 2024-09-14T08:39:16.513 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2024-09-14 08:39:12' 2024-09-14T08:39:16.513 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:39:16.514 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:39:16.514 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:39:16.514 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state_pattern=up+replaying 2024-09-14T08:39:16.514 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local 'prev_last_update=2024-09-14 08:39:12' 2024-09-14T08:39:16.514 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status 2024-09-14T08:39:16.514 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ CEPH_ARGS= 2024-09-14T08:39:16.514 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-14T08:39:16.574 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ status='OKOKOK142174290mirror.1smithi16419.3.0-4775-g7f24e589trueOK42214285mirror.2smithi16419.3.0-4775-g7f24e589falseOK43604363mirror.3smithi16419.3.0-4775-g7f24e589falseOK44194422mirror.0smithi16419.3.0-4775-g7f24e589falseOKtest1159fdda-fb30-49ff-ba1c-60890cdd16a0up+replayingreplaying, {"bytes_per_second":16922419.2,"bytes_per_snapshot":521635840.0,"last_snapshot_bytes":507672576,"last_snapshot_sync_seconds":1,"local_snapshot_timestamp":1726303134,"remote_snapshot_timestamp":1726303134,"replay_state":"idle"}42214285mirror.2smithi1642024-09-14 08:39:12cluster29ab74346-7215-4f9b-943c-26e4f257da03up+stoppedlocal image is primary2024-09-14 08:39:11' 2024-09-14T08:39:16.574 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local last_update state 2024-09-14T08:39:16.574 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2024-09-14T08:39:16.576 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ last_update='2024-09-14 08:39:12' 2024-09-14T08:39:16.576 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2024-09-14T08:39:16.578 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ state=up+replaying 2024-09-14T08:39:16.578 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ echo up+replaying 2024-09-14T08:39:16.578 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep up+replaying 2024-09-14T08:39:16.578 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:up+replaying 2024-09-14T08:39:16.580 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:39:16.580 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:39:16.580 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ snap_name=snap5 2024-09-14T08:39:16.580 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_snap cluster2 mirror test snap5 2024-09-14T08:39:16.580 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:39:16.580 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:39:16.580 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:39:16.580 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap5 2024-09-14T08:39:16.580 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap5 --debug-rbd=20 --debug-journaler=20 2024-09-14T08:39:44.491 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_started cluster1 mirror test 2024-09-14T08:39:44.491 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:39:44.492 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:39:44.492 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:39:44.492 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2024-09-14T08:39:44.492 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:39:44.492 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:39:44.492 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:39:44.492 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=started 2024-09-14T08:39:44.492 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:39:44.492 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:39:44.492 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:39:45.492 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_replay_state cluster1 mirror test started 2024-09-14T08:39:45.492 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:39:45.493 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:39:45.493 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:39:45.493 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=started 2024-09-14T08:39:45.493 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_result 2024-09-14T08:39:45.493 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=stopped 2024-09-14T08:39:45.494 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2024-09-14T08:39:45.494 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_instance=cluster1 2024-09-14T08:39:45.494 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:39:45.494 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ grep -i state 2024-09-14T08:39:45.494 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T08:39:45.494 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance=cluster1 2024-09-14T08:39:45.494 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local loop_instance 2024-09-14T08:39:45.494 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:39:45.495 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sleep 0 2024-09-14T08:39:45.495 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T08:39:45.497 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ seq 0 3 2024-09-14T08:39:45.497 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:39:45.497 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2024-09-14T08:39:45.497 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:0 2024-09-14T08:39:45.497 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:39:45.497 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:39:45.497 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T08:39:45.497 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:0 2024-09-14T08:39:45.498 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:39:45.498 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:39:45.498 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:39:45.498 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=0 2024-09-14T08:39:45.498 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T08:39:45.498 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:39:45.498 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:39:45.498 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=0 2024-09-14T08:39:45.498 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:39:45.498 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T08:39:45.498 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:0 2024-09-14T08:39:45.498 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:39:45.498 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:39:45.498 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T08:39:45.499 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:0 2024-09-14T08:39:45.499 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:39:45.499 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:39:45.499 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:39:45.499 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:39:45.499 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T08:39:45.499 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:39:45.499 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:39:45.499 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=0 2024-09-14T08:39:45.499 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=0 2024-09-14T08:39:45.499 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T08:39:45.519 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:39:45.519 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:39:45.519 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:39:45.519 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2024-09-14T08:39:45.598 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T08:39:45.598 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:39:45.598 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:39:45.601 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:39:45.601 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2024-09-14T08:39:45.601 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:1 2024-09-14T08:39:45.601 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:39:45.601 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:39:45.601 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T08:39:45.601 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:1 2024-09-14T08:39:45.602 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:39:45.602 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:39:45.602 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:39:45.602 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=1 2024-09-14T08:39:45.602 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T08:39:45.602 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:39:45.602 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:39:45.602 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=1 2024-09-14T08:39:45.602 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:39:45.602 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T08:39:45.602 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:1 2024-09-14T08:39:45.602 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:39:45.602 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:39:45.602 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T08:39:45.603 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:1 2024-09-14T08:39:45.603 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:39:45.603 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:39:45.603 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:39:45.603 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:39:45.603 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T08:39:45.603 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:39:45.603 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:39:45.603 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=1 2024-09-14T08:39:45.603 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=1 2024-09-14T08:39:45.603 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T08:39:45.621 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:39:45.621 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:39:45.622 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:39:45.622 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2024-09-14T08:39:45.694 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T08:39:45.694 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:39:45.694 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:39:45.697 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:39:45.697 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2024-09-14T08:39:45.697 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:2 2024-09-14T08:39:45.697 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:39:45.697 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:39:45.697 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-14T08:39:45.697 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:2 2024-09-14T08:39:45.697 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:39:45.698 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:39:45.698 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:39:45.698 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=2 2024-09-14T08:39:45.698 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T08:39:45.698 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:39:45.698 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:39:45.698 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=2 2024-09-14T08:39:45.698 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:39:45.698 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-14T08:39:45.698 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:2 2024-09-14T08:39:45.698 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:39:45.698 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:39:45.698 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T08:39:45.698 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:2 2024-09-14T08:39:45.698 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:39:45.699 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:39:45.699 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:39:45.699 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:39:45.699 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-14T08:39:45.699 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:39:45.699 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:39:45.699 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=2 2024-09-14T08:39:45.699 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=2 2024-09-14T08:39:45.699 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T08:39:45.717 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:39:45.717 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:39:45.717 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:39:45.717 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2024-09-14T08:39:45.796 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ return 0 2024-09-14T08:39:45.797 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T08:39:45.797 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ echo ' "state": "Replaying"' 2024-09-14T08:39:45.797 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep -i Replaying 2024-09-14T08:39:45.800 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: "state": "Replaying" 2024-09-14T08:39:45.802 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_state=started 2024-09-14T08:39:45.802 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test started = started 2024-09-14T08:39:45.802 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:39:45.802 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2024-09-14T08:39:45.802 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:39:45.802 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:39:45.803 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:39:45.803 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:39:45.803 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = journal ']' 2024-09-14T08:39:45.803 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = snapshot ']' 2024-09-14T08:39:45.803 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test 2024-09-14T08:39:45.803 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:39:45.803 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:39:45.803 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:39:45.803 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:39:45.803 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster2-mirror-test.status 2024-09-14T08:39:45.803 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster2-mirror-test.status 2024-09-14T08:39:45.804 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:39:45.804 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-test.status 2024-09-14T08:39:45.804 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster1-mirror-test.status 2024-09-14T08:39:45.804 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster1-mirror-test.status 2024-09-14T08:39:45.804 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:39:45.804 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_status_log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:39:45.804 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ mirror_image_snapshot cluster2 mirror test 2024-09-14T08:39:45.804 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:39:45.804 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:39:45.804 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:39:45.804 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2024-09-14T08:39:47.496 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Snapshot ID: 17 2024-09-14T08:39:47.507 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.status 2024-09-14T08:39:47.508 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:39:47.508 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:39:47.508 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:39:47.508 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-test.status 2024-09-14T08:39:47.508 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2024-09-14T08:39:47.508 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:39:47.580 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-14T08:39:47.583 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snapshot_id=17 2024-09-14T08:39:47.583 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:39:47.583 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:39:47.583 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.2 2024-09-14T08:39:47.784 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:39:47.784 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:39:47.784 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:39:47.785 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:39:47.785 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:39:47.785 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-09-14T08:39:47.785 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:39:47.836 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:39:47.836 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:39:47.837 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:39:47.838 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:39:47.838 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:39:47.838 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:39:47.838 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 17 = '' 2024-09-14T08:39:47.838 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:39:47.838 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.4 2024-09-14T08:39:48.239 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:39:48.240 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:39:48.240 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:39:48.240 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:39:48.240 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:39:48.240 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-09-14T08:39:48.240 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:39:48.292 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:39:48.292 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:39:48.293 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:39:48.293 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:39:48.293 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:39:48.294 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:39:48.294 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 17 = '' 2024-09-14T08:39:48.294 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:39:48.294 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.8 2024-09-14T08:39:49.095 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:39:49.096 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:39:49.096 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:39:49.096 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:39:49.096 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:39:49.096 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-09-14T08:39:49.096 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:39:49.154 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:39:49.154 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:39:49.155 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:39:49.156 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:39:49.156 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:39:49.156 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:39:49.156 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 17 = '' 2024-09-14T08:39:49.156 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:39:49.156 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1.6 2024-09-14T08:39:49.896 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.668652071547957) > weight (0.25) 2024-09-14T08:39:49.896 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.2922220237945098) > weight (0.25) 2024-09-14T08:39:49.896 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.8341450063174404) > weight (0.25) 2024-09-14T08:39:49.896 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.3 with skip (0.8712678343944091) > weight (0.25) 2024-09-14T08:39:49.896 INFO:tasks.rbd_mirror_thrash:waiting for 117 secs before thrashing 2024-09-14T08:39:50.757 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:39:50.758 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:39:50.758 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:39:50.758 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:39:50.758 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:39:50.758 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-09-14T08:39:50.758 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:39:50.850 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:39:50.850 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:39:50.851 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:39:50.851 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:39:50.851 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:39:50.852 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:39:50.852 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 17 = '' 2024-09-14T08:39:50.852 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:39:50.852 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 2 2024-09-14T08:39:52.853 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:39:52.853 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:39:52.854 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:39:52.854 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:39:52.854 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:39:52.854 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-09-14T08:39:52.854 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:39:52.914 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:39:52.916 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=17 2024-09-14T08:39:52.917 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 17 = 17 2024-09-14T08:39:52.917 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:39:52.917 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snap_present cluster1 mirror test snap5 2024-09-14T08:39:52.917 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:39:52.917 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:39:52.917 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:39:52.917 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap5 2024-09-14T08:39:52.917 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:39:52.917 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T08:39:52.917 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:39:53.918 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 info mirror/test@snap5 2024-09-14T08:39:53.968 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'test': 2024-09-14T08:39:54.066 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 512 MiB in 128 objects 2024-09-14T08:39:54.067 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:39:54.067 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 10 2024-09-14T08:39:54.067 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 10ba6324d73b 2024-09-14T08:39:54.067 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.10ba6324d73b 2024-09-14T08:39:54.067 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:39:54.067 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering, non-primary 2024-09-14T08:39:54.067 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:39:54.067 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:39:54.067 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:35:14 2024 2024-09-14T08:39:54.067 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:38:12 2024 2024-09-14T08:39:54.067 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:35:14 2024 2024-09-14T08:39:54.067 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: protected: False 2024-09-14T08:39:54.068 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:39:54.068 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:39:54.068 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: 1159fdda-fb30-49ff-ba1c-60890cdd16a0 2024-09-14T08:39:54.068 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: false 2024-09-14T08:39:54.069 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:39:54.070 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n snap4-clean ']' 2024-09-14T08:39:54.070 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ compare_image_snaps mirror test snap4-clean 2024-09-14T08:39:54.070 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:39:54.070 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:39:54.070 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap4-clean 2024-09-14T08:39:54.070 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local ret=0 2024-09-14T08:39:54.070 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local rmt_export=/tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export 2024-09-14T08:39:54.070 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local loc_export=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:39:54.070 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:39:54.070 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror export test@snap4-clean /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export 2024-09-14T08:39:54.602 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T08:39:54.612 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 -p mirror export test@snap4-clean /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:39:55.423 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T08:39:55.434 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cmp /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:39:55.644 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:39:55.759 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:39:55.760 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ compare_image_snaps mirror test snap5 2024-09-14T08:39:55.760 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:39:55.760 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:39:55.760 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap5 2024-09-14T08:39:55.760 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local ret=0 2024-09-14T08:39:55.760 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local rmt_export=/tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export 2024-09-14T08:39:55.760 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local loc_export=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:39:55.760 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:39:55.760 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror export test@snap5 /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export 2024-09-14T08:39:56.391 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T08:39:56.401 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 -p mirror export test@snap5 /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:39:57.144 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T08:39:57.153 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cmp /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:39:57.364 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:39:57.483 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:39:57.483 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ clean_snap_name=snap5-clean 2024-09-14T08:39:57.483 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_snap cluster2 mirror test snap5-clean 2024-09-14T08:39:57.483 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:39:57.483 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:39:57.483 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:39:57.483 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap5-clean 2024-09-14T08:39:57.483 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap5-clean --debug-rbd=20 --debug-journaler=20 2024-09-14T08:39:58.446 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 10` 2024-09-14T08:39:58.446 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ stress_write_image cluster2 mirror test 2024-09-14T08:39:58.446 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:39:58.446 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:39:58.446 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:39:58.446 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T08:39:58.448 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local duration=8 2024-09-14T08:39:58.448 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set +e 2024-09-14T08:39:58.448 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ timeout 8s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2024-09-14T08:39:58.667 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:100 IOs, 24609792 bytes 2024-09-14T08:39:58.800 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:200 IOs, 43459584 bytes 2024-09-14T08:39:59.045 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:300 IOs, 117880832 bytes 2024-09-14T08:39:59.142 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:400 IOs, 147511296 bytes 2024-09-14T08:39:59.299 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:500 IOs, 190418944 bytes 2024-09-14T08:39:59.409 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:600 IOs, 191048704 bytes 2024-09-14T08:39:59.530 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:700 IOs, 224480256 bytes 2024-09-14T08:39:59.679 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:800 IOs, 258806784 bytes 2024-09-14T08:39:59.889 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:900 IOs, 323601408 bytes 2024-09-14T08:39:59.962 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1000 IOs, 337459200 bytes 2024-09-14T08:40:00.081 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1100 IOs, 356034560 bytes 2024-09-14T08:40:00.269 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1200 IOs, 408822784 bytes 2024-09-14T08:40:00.416 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1300 IOs, 447014912 bytes 2024-09-14T08:40:00.648 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1400 IOs, 530168832 bytes 2024-09-14T08:40:00.807 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1500 IOs, 562483200 bytes 2024-09-14T08:40:00.924 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1600 IOs, 587328512 bytes 2024-09-14T08:40:01.103 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1700 IOs, 639070208 bytes 2024-09-14T08:40:01.263 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1800 IOs, 683065344 bytes 2024-09-14T08:40:01.399 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1900 IOs, 718381056 bytes 2024-09-14T08:40:01.585 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2000 IOs, 778667008 bytes 2024-09-14T08:40:01.728 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2100 IOs, 808969216 bytes 2024-09-14T08:40:01.845 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2200 IOs, 842219520 bytes 2024-09-14T08:40:01.999 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2300 IOs, 879344640 bytes 2024-09-14T08:40:02.127 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2400 IOs, 923242496 bytes 2024-09-14T08:40:02.440 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2500 IOs, 1014366208 bytes 2024-09-14T08:40:02.541 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2600 IOs, 1041406976 bytes 2024-09-14T08:40:02.741 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2700 IOs, 1087284224 bytes 2024-09-14T08:40:02.844 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2800 IOs, 1105315840 bytes 2024-09-14T08:40:02.914 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2900 IOs, 1110794240 bytes 2024-09-14T08:40:03.114 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3000 IOs, 1167924224 bytes 2024-09-14T08:40:03.217 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3100 IOs, 1184776192 bytes 2024-09-14T08:40:03.335 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3200 IOs, 1209351168 bytes 2024-09-14T08:40:03.552 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3300 IOs, 1281891328 bytes 2024-09-14T08:40:03.710 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3400 IOs, 1334385664 bytes 2024-09-14T08:40:03.913 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3500 IOs, 1386417152 bytes 2024-09-14T08:40:04.006 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3600 IOs, 1400665088 bytes 2024-09-14T08:40:04.177 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3700 IOs, 1454267392 bytes 2024-09-14T08:40:04.244 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3800 IOs, 1454913536 bytes 2024-09-14T08:40:04.510 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3900 IOs, 1544673280 bytes 2024-09-14T08:40:04.624 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4000 IOs, 1576455168 bytes 2024-09-14T08:40:04.769 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4100 IOs, 1600626688 bytes 2024-09-14T08:40:04.951 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4200 IOs, 1668400128 bytes 2024-09-14T08:40:05.004 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.9514438829019477) > weight (0.25) 2024-09-14T08:40:05.005 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.7599382058772582) > weight (0.25) 2024-09-14T08:40:05.005 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.9903813921925012) > weight (0.25) 2024-09-14T08:40:05.005 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.3 2024-09-14T08:40:05.005 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Sent signal 15 2024-09-14T08:40:05.005 INFO:tasks.rbd_mirror_thrash:waiting for 1 secs before reviving daemons 2024-09-14T08:40:05.006 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi164.stderr:2024-09-14T08:40:05.006+0000 7f0b8f8b4640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.3 (PID: 39417) UID: 1000 2024-09-14T08:40:05.086 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4300 IOs, 1683867648 bytes 2024-09-14T08:40:05.235 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4400 IOs, 1740715008 bytes 2024-09-14T08:40:05.491 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4500 IOs, 1808579584 bytes 2024-09-14T08:40:05.618 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4600 IOs, 1834975232 bytes 2024-09-14T08:40:05.775 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4700 IOs, 1876419584 bytes 2024-09-14T08:40:05.985 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4800 IOs, 1951727616 bytes 2024-09-14T08:40:06.006 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.3 2024-09-14T08:40:06.006 INFO:teuthology.orchestra.run:waiting for 600 2024-09-14T08:40:06.006 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.3 2024-09-14T08:40:06.007 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Restarting daemon 2024-09-14T08:40:06.007 DEBUG:teuthology.orchestra.run.smithi164:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.3 2024-09-14T08:40:06.009 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Started 2024-09-14T08:40:06.010 INFO:tasks.rbd_mirror_thrash:waiting for 95 secs before thrashing 2024-09-14T08:40:06.143 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4900 IOs, 1972023296 bytes 2024-09-14T08:40:06.288 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5000 IOs, 2006575104 bytes 2024-09-14T08:40:06.453 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ error_code=124 2024-09-14T08:40:06.453 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set -e 2024-09-14T08:40:06.453 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T08:40:06.454 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:40:06.454 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2024-09-14T08:40:06.454 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:40:06.454 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:40:06.454 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:40:06.454 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state_pattern=up+replaying 2024-09-14T08:40:06.454 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local description_pattern= 2024-09-14T08:40:06.454 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local service_pattern= 2024-09-14T08:40:06.454 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:40:06.454 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:40:07.455 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2024-09-14T08:40:07.455 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:40:07.455 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:40:07.455 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:40:07.455 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state_pattern=up+replaying 2024-09-14T08:40:07.455 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local description_pattern= 2024-09-14T08:40:07.455 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local service_pattern= 2024-09-14T08:40:07.455 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status 2024-09-14T08:40:07.455 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ CEPH_ARGS= 2024-09-14T08:40:07.456 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2024-09-14T08:40:07.514 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ status='test: 2024-09-14T08:40:07.514 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: global_id: 1159fdda-fb30-49ff-ba1c-60890cdd16a0 2024-09-14T08:40:07.514 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: state: up+replaying 2024-09-14T08:40:07.514 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":521635840.0,"last_snapshot_bytes":507672576,"last_snapshot_sync_seconds":1,"local_snapshot_timestamp":1726303134,"remote_snapshot_timestamp":1726303134,"replay_state":"idle"} 2024-09-14T08:40:07.514 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: service: mirror.2 on smithi164 2024-09-14T08:40:07.514 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: last_update: 2024-09-14 08:39:42 2024-09-14T08:40:07.514 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: peer_sites: 2024-09-14T08:40:07.514 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: name: cluster2 2024-09-14T08:40:07.514 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: state: up+stopped 2024-09-14T08:40:07.514 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: description: local image is primary 2024-09-14T08:40:07.514 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: last_update: 2024-09-14 08:39:41' 2024-09-14T08:40:07.515 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep '^ state: .*up+replaying' 2024-09-14T08:40:07.515 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: state: up+replaying 2024-09-14T08:40:07.515 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep '^ description: .*' 2024-09-14T08:40:07.516 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":521635840.0,"last_snapshot_bytes":507672576,"last_snapshot_sync_seconds":1,"local_snapshot_timestamp":1726303134,"remote_snapshot_timestamp":1726303134,"replay_state":"idle"} 2024-09-14T08:40:07.516 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n '' ']' 2024-09-14T08:40:07.517 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ echo up+replaying 2024-09-14T08:40:07.517 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep '^up+' 2024-09-14T08:40:07.517 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:up+replaying 2024-09-14T08:40:07.518 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep 'service: *mirror..* on ' 2024-09-14T08:40:07.519 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: service: mirror.2 on smithi164 2024-09-14T08:40:07.519 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local last_update 2024-09-14T08:40:07.519 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-14T08:40:07.521 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ last_update='2024-09-14 08:39:42' 2024-09-14T08:40:07.521 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2024-09-14 08:39:42' 2024-09-14T08:40:07.521 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:40:07.521 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:40:07.522 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:40:07.522 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state_pattern=up+replaying 2024-09-14T08:40:07.522 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local 'prev_last_update=2024-09-14 08:39:42' 2024-09-14T08:40:07.522 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status 2024-09-14T08:40:07.522 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ CEPH_ARGS= 2024-09-14T08:40:07.522 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-14T08:40:07.584 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ status='WARNINGWARNINGOK142174290mirror.1smithi16419.3.0-4775-g7f24e589trueOK42214285mirror.2smithi16419.3.0-4775-g7f24e589falseOK44194422mirror.0smithi16419.3.0-4775-g7f24e589falseOK4487mirror.3smithi16419.3.0-4775-g7f24e589falseWARNINGnot reporting statustest1159fdda-fb30-49ff-ba1c-60890cdd16a0up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":521635840.0,"last_snapshot_bytes":507672576,"last_snapshot_sync_seconds":1,"local_snapshot_timestamp":1726303134,"remote_snapshot_timestamp":1726303134,"replay_state":"idle"}42214285mirror.2smithi1642024-09-14 08:39:42cluster29ab74346-7215-4f9b-943c-26e4f257da03up+stoppedlocal image is primary2024-09-14 08:39:41' 2024-09-14T08:40:07.584 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local last_update state 2024-09-14T08:40:07.584 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2024-09-14T08:40:07.586 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ last_update='2024-09-14 08:39:42' 2024-09-14T08:40:07.586 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2024-09-14T08:40:07.588 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ state=up+replaying 2024-09-14T08:40:07.588 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ echo up+replaying 2024-09-14T08:40:07.588 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep up+replaying 2024-09-14T08:40:07.589 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:up+replaying 2024-09-14T08:40:07.589 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:40:07.589 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:40:07.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ snap_name=snap6 2024-09-14T08:40:07.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_snap cluster2 mirror test snap6 2024-09-14T08:40:07.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:40:07.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:40:07.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:40:07.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap6 2024-09-14T08:40:07.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap6 --debug-rbd=20 --debug-journaler=20 2024-09-14T08:40:35.496 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_started cluster1 mirror test 2024-09-14T08:40:35.496 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:40:35.496 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:40:35.496 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:40:35.496 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2024-09-14T08:40:35.497 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:40:35.497 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:40:35.497 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:40:35.497 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=started 2024-09-14T08:40:35.497 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:40:35.497 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:40:35.497 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:40:36.497 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_replay_state cluster1 mirror test started 2024-09-14T08:40:36.497 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:40:36.497 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:40:36.497 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:40:36.497 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=started 2024-09-14T08:40:36.497 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_result 2024-09-14T08:40:36.497 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=stopped 2024-09-14T08:40:36.498 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2024-09-14T08:40:36.498 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_instance=cluster1 2024-09-14T08:40:36.499 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:40:36.499 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T08:40:36.499 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance=cluster1 2024-09-14T08:40:36.499 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local loop_instance 2024-09-14T08:40:36.499 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ grep -i state 2024-09-14T08:40:36.499 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:40:36.499 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sleep 0 2024-09-14T08:40:36.499 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T08:40:36.499 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ seq 0 3 2024-09-14T08:40:36.501 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:40:36.501 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2024-09-14T08:40:36.501 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:0 2024-09-14T08:40:36.501 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:40:36.501 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:40:36.501 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T08:40:36.501 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:0 2024-09-14T08:40:36.501 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:40:36.501 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:40:36.502 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:40:36.502 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=0 2024-09-14T08:40:36.502 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T08:40:36.502 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:40:36.502 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:40:36.502 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=0 2024-09-14T08:40:36.502 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:40:36.502 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T08:40:36.502 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:0 2024-09-14T08:40:36.502 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:40:36.502 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:40:36.502 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T08:40:36.502 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:0 2024-09-14T08:40:36.503 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:40:36.503 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:40:36.504 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:40:36.504 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:40:36.504 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T08:40:36.504 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:40:36.504 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:40:36.504 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=0 2024-09-14T08:40:36.504 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=0 2024-09-14T08:40:36.504 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T08:40:36.523 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:40:36.523 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:40:36.523 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:40:36.524 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2024-09-14T08:40:36.602 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T08:40:36.602 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:40:36.602 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:40:36.604 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:40:36.605 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2024-09-14T08:40:36.605 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:1 2024-09-14T08:40:36.605 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:40:36.605 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:40:36.605 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T08:40:36.605 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:1 2024-09-14T08:40:36.605 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:40:36.605 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:40:36.605 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:40:36.605 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=1 2024-09-14T08:40:36.605 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T08:40:36.605 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:40:36.606 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:40:36.606 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=1 2024-09-14T08:40:36.606 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:40:36.606 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T08:40:36.606 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:1 2024-09-14T08:40:36.606 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:40:36.606 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:40:36.606 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T08:40:36.606 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:1 2024-09-14T08:40:36.606 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:40:36.606 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:40:36.606 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:40:36.606 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:40:36.606 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T08:40:36.606 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:40:36.607 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:40:36.607 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=1 2024-09-14T08:40:36.607 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=1 2024-09-14T08:40:36.607 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T08:40:36.624 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:40:36.624 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:40:36.624 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:40:36.625 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2024-09-14T08:40:36.697 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T08:40:36.698 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:40:36.698 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:40:36.700 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:40:36.700 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2024-09-14T08:40:36.701 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:2 2024-09-14T08:40:36.701 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:40:36.701 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:40:36.701 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-14T08:40:36.701 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:2 2024-09-14T08:40:36.701 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:40:36.701 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:40:36.701 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:40:36.701 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=2 2024-09-14T08:40:36.701 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T08:40:36.701 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:40:36.701 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:40:36.701 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=2 2024-09-14T08:40:36.702 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:40:36.702 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-14T08:40:36.702 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:2 2024-09-14T08:40:36.702 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:40:36.702 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:40:36.702 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T08:40:36.702 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:2 2024-09-14T08:40:36.702 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:40:36.702 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:40:36.702 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:40:36.702 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:40:36.702 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-14T08:40:36.702 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:40:36.702 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:40:36.703 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=2 2024-09-14T08:40:36.703 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=2 2024-09-14T08:40:36.703 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T08:40:36.719 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:40:36.719 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:40:36.719 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:40:36.719 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2024-09-14T08:40:36.798 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ return 0 2024-09-14T08:40:36.799 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T08:40:36.799 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ echo ' "state": "Replaying"' 2024-09-14T08:40:36.799 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep -i Replaying 2024-09-14T08:40:36.799 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: "state": "Replaying" 2024-09-14T08:40:36.801 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_state=started 2024-09-14T08:40:36.801 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test started = started 2024-09-14T08:40:36.801 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:40:36.801 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2024-09-14T08:40:36.801 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:40:36.801 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:40:36.801 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:40:36.801 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:40:36.802 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = journal ']' 2024-09-14T08:40:36.802 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = snapshot ']' 2024-09-14T08:40:36.802 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test 2024-09-14T08:40:36.802 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:40:36.802 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:40:36.802 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:40:36.802 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:40:36.802 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster2-mirror-test.status 2024-09-14T08:40:36.802 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster2-mirror-test.status 2024-09-14T08:40:36.802 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:40:36.803 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-test.status 2024-09-14T08:40:36.803 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster1-mirror-test.status 2024-09-14T08:40:36.803 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster1-mirror-test.status 2024-09-14T08:40:36.804 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:40:36.804 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_status_log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:40:36.804 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ mirror_image_snapshot cluster2 mirror test 2024-09-14T08:40:36.804 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:40:36.804 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:40:36.804 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:40:36.804 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2024-09-14T08:40:38.505 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Snapshot ID: 20 2024-09-14T08:40:38.515 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.status 2024-09-14T08:40:38.516 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:40:38.516 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:40:38.516 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:40:38.516 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-test.status 2024-09-14T08:40:38.516 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2024-09-14T08:40:38.516 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:40:38.589 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-14T08:40:38.592 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snapshot_id=20 2024-09-14T08:40:38.592 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:40:38.592 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:40:38.592 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.2 2024-09-14T08:40:38.793 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:40:38.794 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:40:38.794 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:40:38.794 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:40:38.794 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:40:38.794 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-09-14T08:40:38.794 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:40:38.844 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:40:38.844 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:40:38.845 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:40:38.845 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:40:38.845 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:40:38.846 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:40:38.846 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 20 = '' 2024-09-14T08:40:38.846 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:40:38.846 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.4 2024-09-14T08:40:39.247 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:40:39.247 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:40:39.247 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:40:39.247 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:40:39.248 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:40:39.248 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-09-14T08:40:39.248 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:40:39.299 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:40:39.299 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:40:39.300 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:40:39.300 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:40:39.301 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:40:39.301 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:40:39.301 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 20 = '' 2024-09-14T08:40:39.301 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:40:39.301 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.8 2024-09-14T08:40:40.102 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:40:40.103 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:40:40.103 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:40:40.103 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:40:40.103 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:40:40.103 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-09-14T08:40:40.103 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:40:40.161 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:40:40.162 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:40:40.163 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:40:40.163 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:40:40.163 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:40:40.164 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:40:40.164 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 20 = '' 2024-09-14T08:40:40.164 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:40:40.164 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1.6 2024-09-14T08:40:41.765 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:40:41.765 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:40:41.765 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:40:41.765 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:40:41.766 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:40:41.766 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-09-14T08:40:41.766 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:40:41.831 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:40:41.831 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:40:41.832 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:40:41.832 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:40:41.832 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:40:41.833 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:40:41.833 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 20 = '' 2024-09-14T08:40:41.833 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:40:41.833 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 2 2024-09-14T08:40:42.508 INFO:tasks.ceph.cluster1.osd.2.smithi135.stderr:2024-09-14T08:40:42.507+0000 7f317814f640 -1 reset not still connected to 0x559cc65a6c30 2024-09-14T08:40:42.508 INFO:tasks.ceph.cluster1.osd.2.smithi135.stderr:2024-09-14T08:40:42.507+0000 7f317814f640 -1 reset not still connected to 0x559cc65a76c0 2024-09-14T08:40:43.835 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:40:43.835 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:40:43.835 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:40:43.835 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:40:43.835 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:40:43.835 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-09-14T08:40:43.836 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:40:43.891 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:40:43.894 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=20 2024-09-14T08:40:43.894 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 20 = 20 2024-09-14T08:40:43.894 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:40:43.894 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snap_present cluster1 mirror test snap6 2024-09-14T08:40:43.894 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:40:43.894 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:40:43.894 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:40:43.894 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap6 2024-09-14T08:40:43.894 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:40:43.894 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T08:40:43.895 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:40:44.895 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 info mirror/test@snap6 2024-09-14T08:40:44.948 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'test': 2024-09-14T08:40:44.948 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 512 MiB in 128 objects 2024-09-14T08:40:44.948 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:40:44.948 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 12 2024-09-14T08:40:44.948 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 10ba6324d73b 2024-09-14T08:40:44.948 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.10ba6324d73b 2024-09-14T08:40:44.948 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:40:44.948 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering, non-primary 2024-09-14T08:40:44.948 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:40:44.948 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:40:44.949 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:35:14 2024 2024-09-14T08:40:44.949 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:39:54 2024 2024-09-14T08:40:44.949 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:35:14 2024 2024-09-14T08:40:44.949 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: protected: False 2024-09-14T08:40:44.949 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:40:44.949 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:40:44.949 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: 1159fdda-fb30-49ff-ba1c-60890cdd16a0 2024-09-14T08:40:44.949 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: false 2024-09-14T08:40:44.952 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:40:44.952 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n snap5-clean ']' 2024-09-14T08:40:44.953 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ compare_image_snaps mirror test snap5-clean 2024-09-14T08:40:44.953 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:40:44.953 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:40:44.953 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap5-clean 2024-09-14T08:40:44.953 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local ret=0 2024-09-14T08:40:44.953 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local rmt_export=/tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export 2024-09-14T08:40:44.953 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local loc_export=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:40:44.953 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:40:44.953 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror export test@snap5-clean /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export 2024-09-14T08:40:45.571 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T08:40:45.579 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 -p mirror export test@snap5-clean /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:40:46.402 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T08:40:46.409 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cmp /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:40:46.621 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:40:46.736 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:40:46.736 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ compare_image_snaps mirror test snap6 2024-09-14T08:40:46.736 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:40:46.736 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:40:46.736 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap6 2024-09-14T08:40:46.736 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local ret=0 2024-09-14T08:40:46.736 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local rmt_export=/tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export 2024-09-14T08:40:46.736 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local loc_export=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:40:46.736 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:40:46.737 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror export test@snap6 /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export 2024-09-14T08:40:47.345 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T08:40:47.353 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 -p mirror export test@snap6 /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:40:48.174 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T08:40:48.182 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cmp /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:40:48.393 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:40:48.511 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:40:48.511 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ clean_snap_name=snap6-clean 2024-09-14T08:40:48.511 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_snap cluster2 mirror test snap6-clean 2024-09-14T08:40:48.512 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:40:48.512 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:40:48.512 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:40:48.512 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap6-clean 2024-09-14T08:40:48.512 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap6-clean --debug-rbd=20 --debug-journaler=20 2024-09-14T08:40:49.458 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 10` 2024-09-14T08:40:49.458 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ stress_write_image cluster2 mirror test 2024-09-14T08:40:49.458 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:40:49.458 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:40:49.458 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:40:49.458 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T08:40:49.460 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local duration=11 2024-09-14T08:40:49.460 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set +e 2024-09-14T08:40:49.460 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ timeout 11s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2024-09-14T08:40:49.791 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:100 IOs, 71974912 bytes 2024-09-14T08:40:50.007 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:200 IOs, 130715648 bytes 2024-09-14T08:40:50.097 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:300 IOs, 132283392 bytes 2024-09-14T08:40:50.280 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:400 IOs, 184707072 bytes 2024-09-14T08:40:50.458 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:500 IOs, 232877056 bytes 2024-09-14T08:40:50.618 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:600 IOs, 283326464 bytes 2024-09-14T08:40:50.699 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:700 IOs, 283930624 bytes 2024-09-14T08:40:50.860 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:800 IOs, 334759936 bytes 2024-09-14T08:40:50.989 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:900 IOs, 362001408 bytes 2024-09-14T08:40:51.089 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1000 IOs, 377697280 bytes 2024-09-14T08:40:51.208 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1100 IOs, 400323584 bytes 2024-09-14T08:40:51.493 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1200 IOs, 495833088 bytes 2024-09-14T08:40:51.806 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1300 IOs, 607660032 bytes 2024-09-14T08:40:51.903 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1400 IOs, 622302208 bytes 2024-09-14T08:40:52.044 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1500 IOs, 661678080 bytes 2024-09-14T08:40:52.174 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1600 IOs, 685137920 bytes 2024-09-14T08:40:52.328 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1700 IOs, 727680000 bytes 2024-09-14T08:40:52.466 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1800 IOs, 767496192 bytes 2024-09-14T08:40:52.565 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1900 IOs, 769436672 bytes 2024-09-14T08:40:52.790 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2000 IOs, 830874624 bytes 2024-09-14T08:40:53.154 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2100 IOs, 948386816 bytes 2024-09-14T08:40:53.275 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2200 IOs, 965593088 bytes 2024-09-14T08:40:53.468 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2300 IOs, 1016449024 bytes 2024-09-14T08:40:53.770 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2400 IOs, 1108601856 bytes 2024-09-14T08:40:53.941 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2500 IOs, 1154954240 bytes 2024-09-14T08:40:54.205 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2600 IOs, 1232369664 bytes 2024-09-14T08:40:54.440 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2700 IOs, 1289688064 bytes 2024-09-14T08:40:54.734 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2800 IOs, 1379753984 bytes 2024-09-14T08:40:54.879 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2900 IOs, 1416073216 bytes 2024-09-14T08:40:54.973 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3000 IOs, 1432998912 bytes 2024-09-14T08:40:55.131 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3100 IOs, 1480850432 bytes 2024-09-14T08:40:55.253 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3200 IOs, 1513996288 bytes 2024-09-14T08:40:55.357 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3300 IOs, 1544893440 bytes 2024-09-14T08:40:55.496 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3400 IOs, 1560678400 bytes 2024-09-14T08:40:55.625 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3500 IOs, 1588339712 bytes 2024-09-14T08:40:55.740 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3600 IOs, 1612125184 bytes 2024-09-14T08:40:55.931 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3700 IOs, 1672272896 bytes 2024-09-14T08:40:56.140 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3800 IOs, 1738749952 bytes 2024-09-14T08:40:56.261 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3900 IOs, 1782925312 bytes 2024-09-14T08:40:56.496 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4000 IOs, 1831849984 bytes 2024-09-14T08:40:56.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4100 IOs, 1848412160 bytes 2024-09-14T08:40:56.757 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4200 IOs, 1899240448 bytes 2024-09-14T08:40:56.858 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4300 IOs, 1914286080 bytes 2024-09-14T08:40:57.066 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4400 IOs, 1976436736 bytes 2024-09-14T08:40:57.192 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4500 IOs, 2005154816 bytes 2024-09-14T08:40:57.350 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4600 IOs, 2057930752 bytes 2024-09-14T08:40:57.533 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4700 IOs, 2095560704 bytes 2024-09-14T08:40:57.617 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4800 IOs, 2096163840 bytes 2024-09-14T08:40:57.705 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4900 IOs, 2109450240 bytes 2024-09-14T08:40:57.995 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5000 IOs, 2209290240 bytes 2024-09-14T08:40:58.208 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5100 IOs, 2279226368 bytes 2024-09-14T08:40:58.318 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5200 IOs, 2294207488 bytes 2024-09-14T08:40:58.428 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5300 IOs, 2314596352 bytes 2024-09-14T08:40:58.558 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5400 IOs, 2354748416 bytes 2024-09-14T08:40:58.673 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5500 IOs, 2373189632 bytes 2024-09-14T08:40:58.843 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5600 IOs, 2421352448 bytes 2024-09-14T08:40:58.953 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5700 IOs, 2440834048 bytes 2024-09-14T08:40:59.024 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5800 IOs, 2445258752 bytes 2024-09-14T08:40:59.183 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5900 IOs, 2486029312 bytes 2024-09-14T08:40:59.269 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6000 IOs, 2507013120 bytes 2024-09-14T08:40:59.491 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6100 IOs, 2567326720 bytes 2024-09-14T08:40:59.596 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6200 IOs, 2584965120 bytes 2024-09-14T08:40:59.835 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6300 IOs, 2667277312 bytes 2024-09-14T08:40:59.962 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6400 IOs, 2701723648 bytes 2024-09-14T08:41:00.142 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6500 IOs, 2751841280 bytes 2024-09-14T08:41:00.289 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6600 IOs, 2785570816 bytes 2024-09-14T08:41:00.359 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6700 IOs, 2792613888 bytes 2024-09-14T08:41:00.466 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ error_code=124 2024-09-14T08:41:00.467 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set -e 2024-09-14T08:41:00.467 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T08:41:00.467 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:41:00.467 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2024-09-14T08:41:00.467 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:41:00.467 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:41:00.467 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:41:00.467 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state_pattern=up+replaying 2024-09-14T08:41:00.467 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local description_pattern= 2024-09-14T08:41:00.467 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local service_pattern= 2024-09-14T08:41:00.467 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:41:00.467 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:41:01.468 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2024-09-14T08:41:01.468 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:41:01.469 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:41:01.469 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:41:01.469 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state_pattern=up+replaying 2024-09-14T08:41:01.469 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local description_pattern= 2024-09-14T08:41:01.469 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local service_pattern= 2024-09-14T08:41:01.469 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status 2024-09-14T08:41:01.469 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ CEPH_ARGS= 2024-09-14T08:41:01.469 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2024-09-14T08:41:01.532 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ status='test: 2024-09-14T08:41:01.532 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: global_id: 1159fdda-fb30-49ff-ba1c-60890cdd16a0 2024-09-14T08:41:01.532 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: state: up+replaying 2024-09-14T08:41:01.532 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: description: replaying, {"bytes_per_second":17158144.0,"bytes_per_snapshot":524729856.0,"last_snapshot_bytes":514744320,"last_snapshot_sync_seconds":1,"local_snapshot_timestamp":1726303237,"remote_snapshot_timestamp":1726303237,"replay_state":"idle","seconds_until_synced":30} 2024-09-14T08:41:01.532 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: service: mirror.2 on smithi164 2024-09-14T08:41:01.532 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: last_update: 2024-09-14 08:40:42 2024-09-14T08:41:01.532 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: peer_sites: 2024-09-14T08:41:01.532 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: name: cluster2 2024-09-14T08:41:01.532 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: state: up+stopped 2024-09-14T08:41:01.532 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: description: local image is primary 2024-09-14T08:41:01.532 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: last_update: 2024-09-14 08:40:41' 2024-09-14T08:41:01.532 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep '^ state: .*up+replaying' 2024-09-14T08:41:01.533 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: state: up+replaying 2024-09-14T08:41:01.533 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep '^ description: .*' 2024-09-14T08:41:01.534 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: description: replaying, {"bytes_per_second":17158144.0,"bytes_per_snapshot":524729856.0,"last_snapshot_bytes":514744320,"last_snapshot_sync_seconds":1,"local_snapshot_timestamp":1726303237,"remote_snapshot_timestamp":1726303237,"replay_state":"idle","seconds_until_synced":30} 2024-09-14T08:41:01.534 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n '' ']' 2024-09-14T08:41:01.535 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ echo up+replaying 2024-09-14T08:41:01.535 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep '^up+' 2024-09-14T08:41:01.535 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:up+replaying 2024-09-14T08:41:01.535 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep 'service: *mirror..* on ' 2024-09-14T08:41:01.537 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: service: mirror.2 on smithi164 2024-09-14T08:41:01.537 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local last_update 2024-09-14T08:41:01.537 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-14T08:41:01.539 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ last_update='2024-09-14 08:40:42' 2024-09-14T08:41:01.539 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2024-09-14 08:40:42' 2024-09-14T08:41:01.539 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:41:01.539 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:41:01.539 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:41:01.540 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state_pattern=up+replaying 2024-09-14T08:41:01.540 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local 'prev_last_update=2024-09-14 08:40:42' 2024-09-14T08:41:01.540 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status 2024-09-14T08:41:01.540 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ CEPH_ARGS= 2024-09-14T08:41:01.540 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-14T08:41:01.602 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ status='OKOKOK142174290mirror.1smithi16419.3.0-4775-g7f24e589trueOK42214285mirror.2smithi16419.3.0-4775-g7f24e589falseOK44194422mirror.0smithi16419.3.0-4775-g7f24e589falseOK44874489mirror.3smithi16419.3.0-4775-g7f24e589falseOKtest1159fdda-fb30-49ff-ba1c-60890cdd16a0up+replayingreplaying, {"bytes_per_second":17158144.0,"bytes_per_snapshot":524729856.0,"last_snapshot_bytes":514744320,"last_snapshot_sync_seconds":1,"local_snapshot_timestamp":1726303237,"remote_snapshot_timestamp":1726303237,"replay_state":"idle","seconds_until_synced":30}42214285mirror.2smithi1642024-09-14 08:40:42cluster29ab74346-7215-4f9b-943c-26e4f257da03up+stoppedlocal image is primary2024-09-14 08:40:41' 2024-09-14T08:41:01.602 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local last_update state 2024-09-14T08:41:01.602 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2024-09-14T08:41:01.604 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ last_update='2024-09-14 08:40:42' 2024-09-14T08:41:01.604 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2024-09-14T08:41:01.606 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ state=up+replaying 2024-09-14T08:41:01.606 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ echo up+replaying 2024-09-14T08:41:01.606 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep up+replaying 2024-09-14T08:41:01.607 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:up+replaying 2024-09-14T08:41:01.607 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:41:01.608 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:41:01.608 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ snap_name=snap7 2024-09-14T08:41:01.608 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_snap cluster2 mirror test snap7 2024-09-14T08:41:01.608 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:41:01.608 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:41:01.608 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:41:01.608 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap7 2024-09-14T08:41:01.608 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap7 --debug-rbd=20 --debug-journaler=20 2024-09-14T08:41:30.495 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_started cluster1 mirror test 2024-09-14T08:41:30.495 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:41:30.496 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:41:30.496 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:41:30.496 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2024-09-14T08:41:30.496 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:41:30.496 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:41:30.496 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:41:30.496 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=started 2024-09-14T08:41:30.496 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:41:30.496 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:41:30.496 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:41:31.496 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_replay_state cluster1 mirror test started 2024-09-14T08:41:31.496 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:41:31.496 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:41:31.496 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:41:31.497 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=started 2024-09-14T08:41:31.497 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_result 2024-09-14T08:41:31.497 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=stopped 2024-09-14T08:41:31.498 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2024-09-14T08:41:31.498 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ grep -i state 2024-09-14T08:41:31.498 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_instance=cluster1 2024-09-14T08:41:31.498 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:41:31.498 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T08:41:31.498 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance=cluster1 2024-09-14T08:41:31.498 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local loop_instance 2024-09-14T08:41:31.498 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:41:31.499 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sleep 0 2024-09-14T08:41:31.499 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T08:41:31.499 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ seq 0 3 2024-09-14T08:41:31.501 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:41:31.502 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2024-09-14T08:41:31.502 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:0 2024-09-14T08:41:31.502 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:41:31.502 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:41:31.502 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T08:41:31.502 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:0 2024-09-14T08:41:31.502 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:41:31.502 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:41:31.502 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:41:31.502 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=0 2024-09-14T08:41:31.502 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T08:41:31.502 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:41:31.502 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:41:31.502 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=0 2024-09-14T08:41:31.503 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:41:31.503 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T08:41:31.503 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:0 2024-09-14T08:41:31.503 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:41:31.503 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:41:31.503 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T08:41:31.503 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:0 2024-09-14T08:41:31.503 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:41:31.503 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:41:31.503 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:41:31.503 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:41:31.503 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T08:41:31.503 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:41:31.503 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:41:31.504 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=0 2024-09-14T08:41:31.504 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=0 2024-09-14T08:41:31.504 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T08:41:31.523 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:41:31.523 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:41:31.523 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:41:31.523 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2024-09-14T08:41:31.603 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T08:41:31.603 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:41:31.603 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:41:31.607 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:41:31.607 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2024-09-14T08:41:31.607 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:1 2024-09-14T08:41:31.607 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:41:31.607 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:41:31.607 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T08:41:31.607 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:1 2024-09-14T08:41:31.607 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:41:31.607 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:41:31.607 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:41:31.607 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=1 2024-09-14T08:41:31.608 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T08:41:31.608 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:41:31.608 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:41:31.608 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=1 2024-09-14T08:41:31.608 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:41:31.608 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T08:41:31.608 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:1 2024-09-14T08:41:31.608 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:41:31.608 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:41:31.608 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T08:41:31.608 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:1 2024-09-14T08:41:31.608 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:41:31.608 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:41:31.608 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:41:31.609 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:41:31.609 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T08:41:31.609 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:41:31.609 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:41:31.609 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=1 2024-09-14T08:41:31.609 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=1 2024-09-14T08:41:31.609 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T08:41:31.625 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:41:31.625 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:41:31.625 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:41:31.625 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2024-09-14T08:41:31.698 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T08:41:31.698 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:41:31.698 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:41:31.701 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:41:31.701 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2024-09-14T08:41:31.701 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:2 2024-09-14T08:41:31.701 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:41:31.701 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:41:31.702 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-14T08:41:31.702 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:2 2024-09-14T08:41:31.702 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:41:31.702 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:41:31.702 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:41:31.702 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=2 2024-09-14T08:41:31.702 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T08:41:31.702 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:41:31.702 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:41:31.702 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=2 2024-09-14T08:41:31.702 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:41:31.702 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-14T08:41:31.702 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:2 2024-09-14T08:41:31.702 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:41:31.703 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:41:31.703 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T08:41:31.703 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:2 2024-09-14T08:41:31.703 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:41:31.703 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:41:31.703 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:41:31.703 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:41:31.703 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-14T08:41:31.703 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:41:31.703 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:41:31.703 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=2 2024-09-14T08:41:31.703 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=2 2024-09-14T08:41:31.703 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T08:41:31.721 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:41:31.721 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:41:31.721 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:41:31.721 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2024-09-14T08:41:31.801 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ return 0 2024-09-14T08:41:31.801 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T08:41:31.801 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ echo ' "state": "Replaying"' 2024-09-14T08:41:31.801 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep -i Replaying 2024-09-14T08:41:31.802 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: "state": "Replaying" 2024-09-14T08:41:31.804 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_state=started 2024-09-14T08:41:31.804 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test started = started 2024-09-14T08:41:31.804 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:41:31.804 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2024-09-14T08:41:31.804 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:41:31.804 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:41:31.804 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:41:31.805 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:41:31.805 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = journal ']' 2024-09-14T08:41:31.805 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = snapshot ']' 2024-09-14T08:41:31.805 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test 2024-09-14T08:41:31.805 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:41:31.805 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:41:31.805 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:41:31.805 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:41:31.805 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster2-mirror-test.status 2024-09-14T08:41:31.805 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster2-mirror-test.status 2024-09-14T08:41:31.805 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:41:31.806 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-test.status 2024-09-14T08:41:31.807 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster1-mirror-test.status 2024-09-14T08:41:31.807 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster1-mirror-test.status 2024-09-14T08:41:31.807 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:41:31.807 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_status_log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:41:31.807 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ mirror_image_snapshot cluster2 mirror test 2024-09-14T08:41:31.807 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:41:31.807 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:41:31.807 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:41:31.807 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2024-09-14T08:41:33.512 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Snapshot ID: 23 2024-09-14T08:41:33.524 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.status 2024-09-14T08:41:33.524 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:41:33.524 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:41:33.524 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:41:33.524 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-test.status 2024-09-14T08:41:33.525 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2024-09-14T08:41:33.525 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:41:33.600 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-14T08:41:33.603 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snapshot_id=23 2024-09-14T08:41:33.603 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:41:33.603 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:41:33.603 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.2 2024-09-14T08:41:33.805 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:41:33.805 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:41:33.805 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:41:33.805 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:41:33.805 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:41:33.805 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-09-14T08:41:33.805 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:41:33.857 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:41:33.857 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:41:33.858 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:41:33.859 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:41:33.859 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:41:33.859 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:41:33.859 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 23 = '' 2024-09-14T08:41:33.860 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:41:33.860 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.4 2024-09-14T08:41:34.261 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:41:34.261 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:41:34.262 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:41:34.262 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:41:34.262 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:41:34.262 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-09-14T08:41:34.262 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:41:34.313 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:41:34.314 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:41:34.315 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:41:34.315 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:41:34.316 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:41:34.316 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:41:34.316 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 23 = '' 2024-09-14T08:41:34.316 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:41:34.316 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.8 2024-09-14T08:41:35.117 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:41:35.117 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:41:35.117 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:41:35.117 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:41:35.117 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:41:35.117 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-09-14T08:41:35.117 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:41:35.175 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:41:35.176 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:41:35.177 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:41:35.177 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:41:35.178 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:41:35.178 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:41:35.178 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 23 = '' 2024-09-14T08:41:35.178 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:41:35.178 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1.6 2024-09-14T08:41:36.779 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:41:36.779 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:41:36.779 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:41:36.779 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:41:36.779 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:41:36.779 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-09-14T08:41:36.779 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:41:36.934 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:41:36.934 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:41:36.936 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:41:36.936 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:41:36.936 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:41:36.936 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:41:36.936 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 23 = '' 2024-09-14T08:41:36.937 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:41:36.937 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 2 2024-09-14T08:41:38.938 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:41:38.938 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:41:38.938 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:41:38.938 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:41:38.938 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:41:38.938 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-09-14T08:41:38.938 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:41:38.996 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:41:38.999 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=23 2024-09-14T08:41:38.999 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 23 = 23 2024-09-14T08:41:38.999 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:41:38.999 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snap_present cluster1 mirror test snap7 2024-09-14T08:41:38.999 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:41:38.999 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:41:38.999 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:41:38.999 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap7 2024-09-14T08:41:38.999 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:41:38.999 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T08:41:38.999 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:41:40.000 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 info mirror/test@snap7 2024-09-14T08:41:40.051 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'test': 2024-09-14T08:41:40.052 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 512 MiB in 128 objects 2024-09-14T08:41:40.052 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:41:40.052 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 14 2024-09-14T08:41:40.052 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 10ba6324d73b 2024-09-14T08:41:40.052 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.10ba6324d73b 2024-09-14T08:41:40.052 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:41:40.052 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering, non-primary 2024-09-14T08:41:40.052 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:41:40.052 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:41:40.052 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:35:14 2024 2024-09-14T08:41:40.052 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:39:54 2024 2024-09-14T08:41:40.052 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:35:14 2024 2024-09-14T08:41:40.052 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: protected: False 2024-09-14T08:41:40.052 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:41:40.053 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:41:40.053 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: 1159fdda-fb30-49ff-ba1c-60890cdd16a0 2024-09-14T08:41:40.053 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: false 2024-09-14T08:41:40.056 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:41:40.056 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n snap6-clean ']' 2024-09-14T08:41:40.056 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ compare_image_snaps mirror test snap6-clean 2024-09-14T08:41:40.056 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:41:40.056 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:41:40.056 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap6-clean 2024-09-14T08:41:40.056 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local ret=0 2024-09-14T08:41:40.056 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local rmt_export=/tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export 2024-09-14T08:41:40.056 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local loc_export=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:41:40.056 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:41:40.057 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror export test@snap6-clean /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export 2024-09-14T08:41:40.693 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T08:41:40.704 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 -p mirror export test@snap6-clean /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:41:41.010 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.0 2024-09-14T08:41:41.011 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Sent signal 15 2024-09-14T08:41:41.011 INFO:tasks.rbd_mirror_thrash:waiting for 0 secs before reviving daemons 2024-09-14T08:41:41.011 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.0 2024-09-14T08:41:41.012 INFO:teuthology.orchestra.run:waiting for 600 2024-09-14T08:41:41.012 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi164.stderr:2024-09-14T08:41:41.010+0000 7fe2406ef640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 (PID: 40294) UID: 1000 2024-09-14T08:41:41.515 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T08:41:41.524 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cmp /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:41:41.733 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:41:41.850 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:41:41.850 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ compare_image_snaps mirror test snap7 2024-09-14T08:41:41.850 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:41:41.850 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:41:41.850 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap7 2024-09-14T08:41:41.850 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local ret=0 2024-09-14T08:41:41.850 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local rmt_export=/tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export 2024-09-14T08:41:41.850 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local loc_export=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:41:41.850 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:41:41.851 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror export test@snap7 /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export 2024-09-14T08:41:42.483 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T08:41:42.493 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 -p mirror export test@snap7 /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:41:43.283 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T08:41:43.292 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cmp /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:41:43.503 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:41:43.619 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:41:43.619 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ clean_snap_name=snap7-clean 2024-09-14T08:41:43.619 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_snap cluster2 mirror test snap7-clean 2024-09-14T08:41:43.619 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:41:43.619 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:41:43.620 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:41:43.620 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap7-clean 2024-09-14T08:41:43.620 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap7-clean --debug-rbd=20 --debug-journaler=20 2024-09-14T08:41:44.472 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 10` 2024-09-14T08:41:44.473 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ stress_write_image cluster2 mirror test 2024-09-14T08:41:44.473 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:41:44.473 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:41:44.473 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:41:44.473 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T08:41:44.475 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local duration=10 2024-09-14T08:41:44.475 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set +e 2024-09-14T08:41:44.475 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ timeout 10s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2024-09-14T08:41:44.674 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:100 IOs, 16031744 bytes 2024-09-14T08:41:44.830 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:200 IOs, 38549504 bytes 2024-09-14T08:41:45.007 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:300 IOs, 81005568 bytes 2024-09-14T08:41:45.239 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:400 IOs, 179501056 bytes 2024-09-14T08:41:45.458 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:500 IOs, 245854208 bytes 2024-09-14T08:41:45.653 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:600 IOs, 298272768 bytes 2024-09-14T08:41:45.833 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:700 IOs, 362509312 bytes 2024-09-14T08:41:46.000 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:800 IOs, 390354944 bytes 2024-09-14T08:41:46.075 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:900 IOs, 391462912 bytes 2024-09-14T08:41:46.256 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1000 IOs, 439526400 bytes 2024-09-14T08:41:46.351 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1100 IOs, 451897344 bytes 2024-09-14T08:41:46.505 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1200 IOs, 489221120 bytes 2024-09-14T08:41:46.612 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1300 IOs, 502010880 bytes 2024-09-14T08:41:46.736 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1400 IOs, 536967168 bytes 2024-09-14T08:41:46.897 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.651961382928485) > weight (0.25) 2024-09-14T08:41:46.897 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.5347491478293597) > weight (0.25) 2024-09-14T08:41:46.897 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.7368531923992385) > weight (0.25) 2024-09-14T08:41:46.897 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.3 with skip (0.6681671043706342) > weight (0.25) 2024-09-14T08:41:46.897 INFO:tasks.rbd_mirror_thrash:waiting for 70 secs before thrashing 2024-09-14T08:41:47.037 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1500 IOs, 626611200 bytes 2024-09-14T08:41:47.114 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.0 2024-09-14T08:41:47.114 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Restarting daemon 2024-09-14T08:41:47.114 DEBUG:teuthology.orchestra.run.smithi164:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 2024-09-14T08:41:47.116 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Started 2024-09-14T08:41:47.117 INFO:tasks.rbd_mirror_thrash:waiting for 111 secs before thrashing 2024-09-14T08:41:47.163 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1600 IOs, 656038912 bytes 2024-09-14T08:41:47.322 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1700 IOs, 693429248 bytes 2024-09-14T08:41:47.495 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1800 IOs, 724225024 bytes 2024-09-14T08:41:47.652 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1900 IOs, 756963328 bytes 2024-09-14T08:41:47.732 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2000 IOs, 764323840 bytes 2024-09-14T08:41:47.854 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2100 IOs, 790831104 bytes 2024-09-14T08:41:48.018 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2200 IOs, 831507456 bytes 2024-09-14T08:41:48.100 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2300 IOs, 838382592 bytes 2024-09-14T08:41:48.293 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2400 IOs, 895953920 bytes 2024-09-14T08:41:48.455 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2500 IOs, 937610240 bytes 2024-09-14T08:41:48.629 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2600 IOs, 989072384 bytes 2024-09-14T08:41:48.879 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2700 IOs, 1058883584 bytes 2024-09-14T08:41:49.041 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2800 IOs, 1097659392 bytes 2024-09-14T08:41:49.201 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2900 IOs, 1139856384 bytes 2024-09-14T08:41:49.437 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3000 IOs, 1211113472 bytes 2024-09-14T08:41:49.509 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3100 IOs, 1211700224 bytes 2024-09-14T08:41:49.666 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3200 IOs, 1243658240 bytes 2024-09-14T08:41:49.784 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3300 IOs, 1269792768 bytes 2024-09-14T08:41:49.882 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3400 IOs, 1278737408 bytes 2024-09-14T08:41:50.011 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3500 IOs, 1325474816 bytes 2024-09-14T08:41:50.196 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3600 IOs, 1362146304 bytes 2024-09-14T08:41:50.285 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3700 IOs, 1364329472 bytes 2024-09-14T08:41:50.465 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3800 IOs, 1406940160 bytes 2024-09-14T08:41:50.792 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3900 IOs, 1518756864 bytes 2024-09-14T08:41:51.008 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4000 IOs, 1583559680 bytes 2024-09-14T08:41:51.239 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4100 IOs, 1655275520 bytes 2024-09-14T08:41:51.391 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4200 IOs, 1695635456 bytes 2024-09-14T08:41:51.511 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4300 IOs, 1727133696 bytes 2024-09-14T08:41:51.704 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4400 IOs, 1779187712 bytes 2024-09-14T08:41:51.826 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4500 IOs, 1804712960 bytes 2024-09-14T08:41:52.143 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4600 IOs, 1910346752 bytes 2024-09-14T08:41:52.329 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4700 IOs, 1962533888 bytes 2024-09-14T08:41:52.526 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4800 IOs, 2019249152 bytes 2024-09-14T08:41:52.630 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4900 IOs, 2042263552 bytes 2024-09-14T08:41:52.817 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5000 IOs, 2092346368 bytes 2024-09-14T08:41:53.058 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5100 IOs, 2162270208 bytes 2024-09-14T08:41:53.244 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5200 IOs, 2233318400 bytes 2024-09-14T08:41:53.505 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5300 IOs, 2296640512 bytes 2024-09-14T08:41:53.719 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5400 IOs, 2379558912 bytes 2024-09-14T08:41:53.875 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5500 IOs, 2402295808 bytes 2024-09-14T08:41:53.994 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5600 IOs, 2421291008 bytes 2024-09-14T08:41:54.230 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5700 IOs, 2494299136 bytes 2024-09-14T08:41:54.304 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5800 IOs, 2496970752 bytes 2024-09-14T08:41:54.454 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5900 IOs, 2536627200 bytes 2024-09-14T08:41:54.480 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ error_code=124 2024-09-14T08:41:54.481 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set -e 2024-09-14T08:41:54.481 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T08:41:54.481 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:41:54.481 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2024-09-14T08:41:54.481 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:41:54.481 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:41:54.481 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:41:54.481 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state_pattern=up+replaying 2024-09-14T08:41:54.481 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local description_pattern= 2024-09-14T08:41:54.481 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local service_pattern= 2024-09-14T08:41:54.481 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:41:54.481 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:41:55.482 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2024-09-14T08:41:55.482 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:41:55.482 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:41:55.482 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:41:55.482 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state_pattern=up+replaying 2024-09-14T08:41:55.482 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local description_pattern= 2024-09-14T08:41:55.482 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local service_pattern= 2024-09-14T08:41:55.482 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status 2024-09-14T08:41:55.483 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ CEPH_ARGS= 2024-09-14T08:41:55.483 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2024-09-14T08:41:55.539 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ status='test: 2024-09-14T08:41:55.539 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: global_id: 1159fdda-fb30-49ff-ba1c-60890cdd16a0 2024-09-14T08:41:55.539 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: state: up+replaying 2024-09-14T08:41:55.539 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: description: replaying, {"bytes_per_second":17676697.6,"bytes_per_snapshot":522522624.0,"last_snapshot_bytes":530300928,"last_snapshot_sync_seconds":1,"local_snapshot_timestamp":1726303292,"remote_snapshot_timestamp":1726303292,"replay_state":"idle"} 2024-09-14T08:41:55.539 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: service: mirror.2 on smithi164 2024-09-14T08:41:55.539 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: last_update: 2024-09-14 08:41:42 2024-09-14T08:41:55.539 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: peer_sites: 2024-09-14T08:41:55.539 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: name: cluster2 2024-09-14T08:41:55.539 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: state: up+stopped 2024-09-14T08:41:55.539 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: description: local image is primary 2024-09-14T08:41:55.539 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: last_update: 2024-09-14 08:41:41' 2024-09-14T08:41:55.539 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep '^ state: .*up+replaying' 2024-09-14T08:41:55.540 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: state: up+replaying 2024-09-14T08:41:55.540 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep '^ description: .*' 2024-09-14T08:41:55.541 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: description: replaying, {"bytes_per_second":17676697.6,"bytes_per_snapshot":522522624.0,"last_snapshot_bytes":530300928,"last_snapshot_sync_seconds":1,"local_snapshot_timestamp":1726303292,"remote_snapshot_timestamp":1726303292,"replay_state":"idle"} 2024-09-14T08:41:55.541 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n '' ']' 2024-09-14T08:41:55.541 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ echo up+replaying 2024-09-14T08:41:55.541 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep '^up+' 2024-09-14T08:41:55.542 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:up+replaying 2024-09-14T08:41:55.542 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep 'service: *mirror..* on ' 2024-09-14T08:41:55.543 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: service: mirror.2 on smithi164 2024-09-14T08:41:55.543 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local last_update 2024-09-14T08:41:55.544 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-14T08:41:55.545 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ last_update='2024-09-14 08:41:42' 2024-09-14T08:41:55.546 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2024-09-14 08:41:42' 2024-09-14T08:41:55.546 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:41:55.546 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:41:55.546 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:41:55.546 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state_pattern=up+replaying 2024-09-14T08:41:55.546 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local 'prev_last_update=2024-09-14 08:41:42' 2024-09-14T08:41:55.546 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status 2024-09-14T08:41:55.546 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ CEPH_ARGS= 2024-09-14T08:41:55.546 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-14T08:41:55.607 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ status='OKOKOK142174290mirror.1smithi16419.3.0-4775-g7f24e589trueOK42214285mirror.2smithi16419.3.0-4775-g7f24e589falseOK44874489mirror.3smithi16419.3.0-4775-g7f24e589falseOK45544557mirror.0smithi16419.3.0-4775-g7f24e589falseOKtest1159fdda-fb30-49ff-ba1c-60890cdd16a0up+replayingreplaying, {"bytes_per_second":17676697.6,"bytes_per_snapshot":522522624.0,"last_snapshot_bytes":530300928,"last_snapshot_sync_seconds":1,"local_snapshot_timestamp":1726303292,"remote_snapshot_timestamp":1726303292,"replay_state":"idle"}42214285mirror.2smithi1642024-09-14 08:41:42cluster29ab74346-7215-4f9b-943c-26e4f257da03up+stoppedlocal image is primary2024-09-14 08:41:41' 2024-09-14T08:41:55.607 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local last_update state 2024-09-14T08:41:55.607 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2024-09-14T08:41:55.609 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ last_update='2024-09-14 08:41:42' 2024-09-14T08:41:55.609 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2024-09-14T08:41:55.611 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ state=up+replaying 2024-09-14T08:41:55.611 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ echo up+replaying 2024-09-14T08:41:55.611 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep up+replaying 2024-09-14T08:41:55.612 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:up+replaying 2024-09-14T08:41:55.613 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:41:55.613 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:41:55.613 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ snap_name=snap8 2024-09-14T08:41:55.613 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_snap cluster2 mirror test snap8 2024-09-14T08:41:55.613 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:41:55.613 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:41:55.613 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:41:55.613 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap8 2024-09-14T08:41:55.613 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap8 --debug-rbd=20 --debug-journaler=20 2024-09-14T08:42:22.515 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_started cluster1 mirror test 2024-09-14T08:42:22.515 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:42:22.515 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:42:22.515 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:42:22.515 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2024-09-14T08:42:22.515 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:42:22.515 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:42:22.515 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:42:22.515 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=started 2024-09-14T08:42:22.515 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:42:22.516 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:42:22.516 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:42:23.516 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_replay_state cluster1 mirror test started 2024-09-14T08:42:23.516 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:42:23.516 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:42:23.516 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:42:23.516 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=started 2024-09-14T08:42:23.516 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_result 2024-09-14T08:42:23.516 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=stopped 2024-09-14T08:42:23.517 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2024-09-14T08:42:23.517 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_instance=cluster1 2024-09-14T08:42:23.517 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ grep -i state 2024-09-14T08:42:23.517 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:42:23.517 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T08:42:23.517 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance=cluster1 2024-09-14T08:42:23.517 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local loop_instance 2024-09-14T08:42:23.518 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:42:23.518 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sleep 0 2024-09-14T08:42:23.518 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T08:42:23.520 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ seq 0 3 2024-09-14T08:42:23.520 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:42:23.520 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2024-09-14T08:42:23.520 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:0 2024-09-14T08:42:23.520 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:42:23.520 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:42:23.520 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T08:42:23.520 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:0 2024-09-14T08:42:23.520 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:42:23.520 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:42:23.520 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:42:23.521 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=0 2024-09-14T08:42:23.521 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T08:42:23.521 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:42:23.521 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:42:23.521 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=0 2024-09-14T08:42:23.521 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:42:23.521 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T08:42:23.521 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:0 2024-09-14T08:42:23.521 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:42:23.521 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:42:23.521 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T08:42:23.521 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:0 2024-09-14T08:42:23.521 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:42:23.522 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:42:23.522 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:42:23.522 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:42:23.523 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T08:42:23.523 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:42:23.523 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:42:23.523 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=0 2024-09-14T08:42:23.523 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=0 2024-09-14T08:42:23.523 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T08:42:23.542 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:42:23.542 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:42:23.542 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:42:23.542 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2024-09-14T08:42:23.621 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T08:42:23.621 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:42:23.621 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:42:23.624 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:42:23.624 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2024-09-14T08:42:23.624 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:1 2024-09-14T08:42:23.624 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:42:23.624 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:42:23.624 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T08:42:23.625 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:1 2024-09-14T08:42:23.625 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:42:23.625 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:42:23.625 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:42:23.625 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=1 2024-09-14T08:42:23.625 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T08:42:23.625 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:42:23.625 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:42:23.625 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=1 2024-09-14T08:42:23.625 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:42:23.625 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T08:42:23.625 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:1 2024-09-14T08:42:23.625 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:42:23.625 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:42:23.625 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T08:42:23.626 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:1 2024-09-14T08:42:23.626 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:42:23.626 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:42:23.626 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:42:23.626 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:42:23.626 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T08:42:23.626 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:42:23.626 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:42:23.626 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=1 2024-09-14T08:42:23.626 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=1 2024-09-14T08:42:23.626 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T08:42:23.644 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:42:23.644 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:42:23.645 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:42:23.645 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2024-09-14T08:42:23.718 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T08:42:23.718 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:42:23.718 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:42:23.721 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:42:23.721 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2024-09-14T08:42:23.721 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:2 2024-09-14T08:42:23.721 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:42:23.721 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:42:23.721 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-14T08:42:23.721 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:2 2024-09-14T08:42:23.721 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:42:23.722 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:42:23.722 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:42:23.722 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=2 2024-09-14T08:42:23.722 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T08:42:23.722 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:42:23.722 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:42:23.722 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=2 2024-09-14T08:42:23.722 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:42:23.722 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-14T08:42:23.722 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:2 2024-09-14T08:42:23.722 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:42:23.722 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:42:23.722 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T08:42:23.722 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:2 2024-09-14T08:42:23.722 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:42:23.723 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:42:23.723 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:42:23.723 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:42:23.723 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-14T08:42:23.723 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:42:23.723 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:42:23.723 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=2 2024-09-14T08:42:23.723 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=2 2024-09-14T08:42:23.723 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T08:42:23.741 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:42:23.741 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:42:23.741 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:42:23.741 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2024-09-14T08:42:23.821 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ return 0 2024-09-14T08:42:23.822 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T08:42:23.822 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ echo ' "state": "Replaying"' 2024-09-14T08:42:23.822 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep -i Replaying 2024-09-14T08:42:23.823 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: "state": "Replaying" 2024-09-14T08:42:23.824 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_state=started 2024-09-14T08:42:23.825 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test started = started 2024-09-14T08:42:23.825 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:42:23.825 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2024-09-14T08:42:23.825 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:42:23.825 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:42:23.825 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:42:23.825 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:42:23.825 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = journal ']' 2024-09-14T08:42:23.825 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = snapshot ']' 2024-09-14T08:42:23.825 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test 2024-09-14T08:42:23.825 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:42:23.825 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:42:23.825 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:42:23.825 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:42:23.826 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster2-mirror-test.status 2024-09-14T08:42:23.826 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster2-mirror-test.status 2024-09-14T08:42:23.826 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:42:23.827 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-test.status 2024-09-14T08:42:23.827 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster1-mirror-test.status 2024-09-14T08:42:23.827 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster1-mirror-test.status 2024-09-14T08:42:23.827 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:42:23.827 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_status_log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:42:23.827 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ mirror_image_snapshot cluster2 mirror test 2024-09-14T08:42:23.827 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:42:23.827 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:42:23.827 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:42:23.828 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2024-09-14T08:42:25.527 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Snapshot ID: 26 2024-09-14T08:42:25.539 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.status 2024-09-14T08:42:25.539 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:42:25.540 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:42:25.540 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:42:25.540 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-test.status 2024-09-14T08:42:25.540 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2024-09-14T08:42:25.540 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:42:25.614 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-14T08:42:25.617 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snapshot_id=26 2024-09-14T08:42:25.617 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:42:25.617 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:42:25.617 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.2 2024-09-14T08:42:25.818 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:42:25.819 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:42:25.819 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:42:25.819 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:42:25.819 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:42:25.819 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-09-14T08:42:25.819 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:42:25.872 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:42:25.872 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:42:25.873 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:42:25.873 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:42:25.873 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:42:25.874 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:42:25.874 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 26 = '' 2024-09-14T08:42:25.874 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:42:25.874 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.4 2024-09-14T08:42:26.275 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:42:26.275 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:42:26.276 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:42:26.276 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:42:26.276 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:42:26.276 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-09-14T08:42:26.276 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:42:26.327 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:42:26.327 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:42:26.329 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:42:26.329 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:42:26.329 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:42:26.329 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:42:26.329 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 26 = '' 2024-09-14T08:42:26.330 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:42:26.330 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.8 2024-09-14T08:42:27.131 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:42:27.131 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:42:27.131 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:42:27.131 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:42:27.131 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:42:27.131 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-09-14T08:42:27.131 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:42:27.185 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:42:27.186 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:42:27.187 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:42:27.188 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:42:27.188 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:42:27.188 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:42:27.188 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 26 = '' 2024-09-14T08:42:27.188 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:42:27.189 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1.6 2024-09-14T08:42:28.790 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:42:28.790 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:42:28.790 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:42:28.790 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:42:28.790 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:42:28.790 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-09-14T08:42:28.790 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:42:28.843 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:42:28.845 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=26 2024-09-14T08:42:28.845 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 26 = 26 2024-09-14T08:42:28.846 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:42:28.846 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snap_present cluster1 mirror test snap8 2024-09-14T08:42:28.846 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:42:28.846 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:42:28.846 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:42:28.846 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap8 2024-09-14T08:42:28.846 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:42:28.846 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T08:42:28.846 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:42:29.846 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 info mirror/test@snap8 2024-09-14T08:42:29.895 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'test': 2024-09-14T08:42:29.895 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 512 MiB in 128 objects 2024-09-14T08:42:29.896 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:42:29.896 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 16 2024-09-14T08:42:29.896 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 10ba6324d73b 2024-09-14T08:42:29.896 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.10ba6324d73b 2024-09-14T08:42:29.896 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:42:29.896 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering, non-primary 2024-09-14T08:42:29.896 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:42:29.896 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:42:29.896 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:35:14 2024 2024-09-14T08:42:29.896 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:41:40 2024 2024-09-14T08:42:29.896 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:35:14 2024 2024-09-14T08:42:29.896 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: protected: False 2024-09-14T08:42:29.897 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:42:29.897 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:42:29.897 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: 1159fdda-fb30-49ff-ba1c-60890cdd16a0 2024-09-14T08:42:29.897 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: false 2024-09-14T08:42:29.900 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:42:29.900 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n snap7-clean ']' 2024-09-14T08:42:29.900 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ compare_image_snaps mirror test snap7-clean 2024-09-14T08:42:29.900 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:42:29.900 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:42:29.900 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap7-clean 2024-09-14T08:42:29.900 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local ret=0 2024-09-14T08:42:29.901 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local rmt_export=/tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export 2024-09-14T08:42:29.901 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local loc_export=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:42:29.901 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:42:29.901 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror export test@snap7-clean /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export 2024-09-14T08:42:30.542 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T08:42:30.553 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 -p mirror export test@snap7-clean /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:42:31.410 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T08:42:31.418 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cmp /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:42:31.628 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:42:31.742 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:42:31.742 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ compare_image_snaps mirror test snap8 2024-09-14T08:42:31.742 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:42:31.742 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:42:31.743 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap8 2024-09-14T08:42:31.743 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local ret=0 2024-09-14T08:42:31.743 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local rmt_export=/tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export 2024-09-14T08:42:31.743 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local loc_export=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:42:31.743 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:42:31.743 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror export test@snap8 /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export 2024-09-14T08:42:32.394 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T08:42:32.404 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 -p mirror export test@snap8 /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:42:33.179 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T08:42:33.187 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cmp /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:42:33.398 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:42:33.516 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:42:33.516 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ clean_snap_name=snap8-clean 2024-09-14T08:42:33.516 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_snap cluster2 mirror test snap8-clean 2024-09-14T08:42:33.516 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:42:33.516 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:42:33.516 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:42:33.516 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap8-clean 2024-09-14T08:42:33.516 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap8-clean --debug-rbd=20 --debug-journaler=20 2024-09-14T08:42:34.473 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 10` 2024-09-14T08:42:34.473 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ stress_write_image cluster2 mirror test 2024-09-14T08:42:34.473 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:42:34.474 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:42:34.474 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:42:34.474 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T08:42:34.475 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local duration=11 2024-09-14T08:42:34.475 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set +e 2024-09-14T08:42:34.475 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ timeout 11s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2024-09-14T08:42:34.795 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:100 IOs, 62801920 bytes 2024-09-14T08:42:35.025 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:200 IOs, 117006336 bytes 2024-09-14T08:42:35.235 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:300 IOs, 179535872 bytes 2024-09-14T08:42:35.397 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:400 IOs, 212716544 bytes 2024-09-14T08:42:35.533 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:500 IOs, 243897344 bytes 2024-09-14T08:42:35.649 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:600 IOs, 265835520 bytes 2024-09-14T08:42:35.840 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:700 IOs, 322919424 bytes 2024-09-14T08:42:36.045 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:800 IOs, 393228288 bytes 2024-09-14T08:42:36.249 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:900 IOs, 455367680 bytes 2024-09-14T08:42:36.344 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1000 IOs, 466590720 bytes 2024-09-14T08:42:36.578 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1100 IOs, 530130944 bytes 2024-09-14T08:42:36.775 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1200 IOs, 589412352 bytes 2024-09-14T08:42:36.960 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1300 IOs, 643545088 bytes 2024-09-14T08:42:37.043 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1400 IOs, 649210880 bytes 2024-09-14T08:42:37.154 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1500 IOs, 670127104 bytes 2024-09-14T08:42:37.360 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1600 IOs, 730812416 bytes 2024-09-14T08:42:37.452 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1700 IOs, 743946240 bytes 2024-09-14T08:42:37.597 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1800 IOs, 776370176 bytes 2024-09-14T08:42:37.709 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1900 IOs, 793423872 bytes 2024-09-14T08:42:37.859 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2000 IOs, 831392768 bytes 2024-09-14T08:42:38.002 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2100 IOs, 858181632 bytes 2024-09-14T08:42:38.099 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2200 IOs, 870938624 bytes 2024-09-14T08:42:38.211 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2300 IOs, 891409408 bytes 2024-09-14T08:42:38.392 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2400 IOs, 954029056 bytes 2024-09-14T08:42:38.533 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2500 IOs, 973923328 bytes 2024-09-14T08:42:38.655 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2600 IOs, 1020280832 bytes 2024-09-14T08:42:38.841 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2700 IOs, 1057375232 bytes 2024-09-14T08:42:38.944 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2800 IOs, 1066280960 bytes 2024-09-14T08:42:39.064 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2900 IOs, 1089899520 bytes 2024-09-14T08:42:39.240 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3000 IOs, 1144902656 bytes 2024-09-14T08:42:39.385 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3100 IOs, 1167581184 bytes 2024-09-14T08:42:39.641 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3200 IOs, 1247366144 bytes 2024-09-14T08:42:39.753 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3300 IOs, 1266624512 bytes 2024-09-14T08:42:40.023 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3400 IOs, 1365187584 bytes 2024-09-14T08:42:40.154 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3500 IOs, 1382085632 bytes 2024-09-14T08:42:40.275 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3600 IOs, 1411301376 bytes 2024-09-14T08:42:40.458 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3700 IOs, 1459886080 bytes 2024-09-14T08:42:40.602 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3800 IOs, 1490220032 bytes 2024-09-14T08:42:40.706 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3900 IOs, 1508990976 bytes 2024-09-14T08:42:40.945 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4000 IOs, 1583072256 bytes 2024-09-14T08:42:41.114 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4100 IOs, 1628414976 bytes 2024-09-14T08:42:41.278 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4200 IOs, 1685473280 bytes 2024-09-14T08:42:41.424 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4300 IOs, 1711001600 bytes 2024-09-14T08:42:41.633 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4400 IOs, 1775113216 bytes 2024-09-14T08:42:41.948 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4500 IOs, 1907892224 bytes 2024-09-14T08:42:42.116 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4600 IOs, 1930596352 bytes 2024-09-14T08:42:42.281 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4700 IOs, 1968529408 bytes 2024-09-14T08:42:42.350 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4800 IOs, 1969134592 bytes 2024-09-14T08:42:42.579 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4900 IOs, 2043332608 bytes 2024-09-14T08:42:42.762 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5000 IOs, 2089908224 bytes 2024-09-14T08:42:42.971 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5100 IOs, 2150776832 bytes 2024-09-14T08:42:43.143 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5200 IOs, 2196798464 bytes 2024-09-14T08:42:43.293 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5300 IOs, 2223483904 bytes 2024-09-14T08:42:43.522 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5400 IOs, 2290311168 bytes 2024-09-14T08:42:43.660 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5500 IOs, 2320018432 bytes 2024-09-14T08:42:43.783 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5600 IOs, 2347912192 bytes 2024-09-14T08:42:43.983 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5700 IOs, 2433427456 bytes 2024-09-14T08:42:44.202 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5800 IOs, 2471919616 bytes 2024-09-14T08:42:44.320 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5900 IOs, 2485581824 bytes 2024-09-14T08:42:44.505 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6000 IOs, 2541467648 bytes 2024-09-14T08:42:44.642 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6100 IOs, 2570291200 bytes 2024-09-14T08:42:44.787 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6200 IOs, 2607192064 bytes 2024-09-14T08:42:44.968 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6300 IOs, 2664593408 bytes 2024-09-14T08:42:45.120 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6400 IOs, 2698527744 bytes 2024-09-14T08:42:45.243 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6500 IOs, 2734065664 bytes 2024-09-14T08:42:45.461 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6600 IOs, 2795723776 bytes 2024-09-14T08:42:45.481 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ error_code=124 2024-09-14T08:42:45.481 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set -e 2024-09-14T08:42:45.481 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T08:42:45.481 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:42:45.482 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2024-09-14T08:42:45.482 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:42:45.482 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:42:45.482 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:42:45.482 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state_pattern=up+replaying 2024-09-14T08:42:45.482 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local description_pattern= 2024-09-14T08:42:45.483 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local service_pattern= 2024-09-14T08:42:45.483 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:42:45.483 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:42:46.482 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2024-09-14T08:42:46.482 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:42:46.483 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:42:46.483 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:42:46.483 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state_pattern=up+replaying 2024-09-14T08:42:46.483 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local description_pattern= 2024-09-14T08:42:46.483 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local service_pattern= 2024-09-14T08:42:46.483 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status 2024-09-14T08:42:46.483 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ CEPH_ARGS= 2024-09-14T08:42:46.483 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2024-09-14T08:42:46.543 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ status='test: 2024-09-14T08:42:46.543 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: global_id: 1159fdda-fb30-49ff-ba1c-60890cdd16a0 2024-09-14T08:42:46.543 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: state: up+replaying 2024-09-14T08:42:46.543 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: description: replaying, {"bytes_per_second":17777902.93,"bytes_per_snapshot":531819008.0,"last_snapshot_bytes":533337088,"last_snapshot_sync_seconds":1,"local_snapshot_timestamp":1726303344,"remote_snapshot_timestamp":1726303344,"replay_state":"idle"} 2024-09-14T08:42:46.544 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: service: mirror.2 on smithi164 2024-09-14T08:42:46.544 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: last_update: 2024-09-14 08:42:42 2024-09-14T08:42:46.544 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: peer_sites: 2024-09-14T08:42:46.544 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: name: cluster2 2024-09-14T08:42:46.544 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: state: up+stopped 2024-09-14T08:42:46.544 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: description: local image is primary 2024-09-14T08:42:46.544 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: last_update: 2024-09-14 08:42:41' 2024-09-14T08:42:46.544 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep '^ state: .*up+replaying' 2024-09-14T08:42:46.545 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: state: up+replaying 2024-09-14T08:42:46.545 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep '^ description: .*' 2024-09-14T08:42:46.546 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: description: replaying, {"bytes_per_second":17777902.93,"bytes_per_snapshot":531819008.0,"last_snapshot_bytes":533337088,"last_snapshot_sync_seconds":1,"local_snapshot_timestamp":1726303344,"remote_snapshot_timestamp":1726303344,"replay_state":"idle"} 2024-09-14T08:42:46.546 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n '' ']' 2024-09-14T08:42:46.546 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ echo up+replaying 2024-09-14T08:42:46.546 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep '^up+' 2024-09-14T08:42:46.547 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:up+replaying 2024-09-14T08:42:46.547 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep 'service: *mirror..* on ' 2024-09-14T08:42:46.548 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: service: mirror.2 on smithi164 2024-09-14T08:42:46.548 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local last_update 2024-09-14T08:42:46.548 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-14T08:42:46.551 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ last_update='2024-09-14 08:42:42' 2024-09-14T08:42:46.551 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2024-09-14 08:42:42' 2024-09-14T08:42:46.551 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:42:46.551 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:42:46.551 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:42:46.551 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state_pattern=up+replaying 2024-09-14T08:42:46.551 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local 'prev_last_update=2024-09-14 08:42:42' 2024-09-14T08:42:46.551 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status 2024-09-14T08:42:46.551 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ CEPH_ARGS= 2024-09-14T08:42:46.551 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-14T08:42:46.614 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ status='OKOKOK142174290mirror.1smithi16419.3.0-4775-g7f24e589trueOK42214285mirror.2smithi16419.3.0-4775-g7f24e589falseOK44874489mirror.3smithi16419.3.0-4775-g7f24e589falseOK45544557mirror.0smithi16419.3.0-4775-g7f24e589falseOKtest1159fdda-fb30-49ff-ba1c-60890cdd16a0up+replayingreplaying, {"bytes_per_second":17777902.93,"bytes_per_snapshot":531819008.0,"last_snapshot_bytes":533337088,"last_snapshot_sync_seconds":1,"local_snapshot_timestamp":1726303344,"remote_snapshot_timestamp":1726303344,"replay_state":"idle"}42214285mirror.2smithi1642024-09-14 08:42:42cluster29ab74346-7215-4f9b-943c-26e4f257da03up+stoppedlocal image is primary2024-09-14 08:42:41' 2024-09-14T08:42:46.614 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local last_update state 2024-09-14T08:42:46.614 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2024-09-14T08:42:46.616 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ last_update='2024-09-14 08:42:42' 2024-09-14T08:42:46.616 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2024-09-14T08:42:46.618 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ state=up+replaying 2024-09-14T08:42:46.618 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ echo up+replaying 2024-09-14T08:42:46.618 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep up+replaying 2024-09-14T08:42:46.619 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:up+replaying 2024-09-14T08:42:46.620 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:42:46.620 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:42:46.620 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ snap_name=snap9 2024-09-14T08:42:46.620 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_snap cluster2 mirror test snap9 2024-09-14T08:42:46.620 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:42:46.620 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:42:46.620 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:42:46.620 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap9 2024-09-14T08:42:46.620 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap9 --debug-rbd=20 --debug-journaler=20 2024-09-14T08:42:56.898 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.9305772688823637) > weight (0.25) 2024-09-14T08:42:56.898 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.1 2024-09-14T08:42:56.898 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Sent signal 15 2024-09-14T08:42:56.899 INFO:tasks.rbd_mirror_thrash:waiting for 9 secs before reviving daemons 2024-09-14T08:42:56.900 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi164.stderr:2024-09-14T08:42:56.899+0000 7fb10ff4a640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 (PID: 36863) UID: 1000 2024-09-14T08:43:05.899 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.1 2024-09-14T08:43:05.899 INFO:teuthology.orchestra.run:waiting for 600 2024-09-14T08:43:05.900 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.1 2024-09-14T08:43:05.900 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Restarting daemon 2024-09-14T08:43:05.900 DEBUG:teuthology.orchestra.run.smithi164:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 2024-09-14T08:43:05.903 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Started 2024-09-14T08:43:05.903 INFO:tasks.rbd_mirror_thrash:waiting for 93 secs before thrashing 2024-09-14T08:43:15.523 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_started cluster1 mirror test 2024-09-14T08:43:15.524 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:43:15.524 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:43:15.524 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:43:15.524 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2024-09-14T08:43:15.524 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:43:15.524 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:43:15.524 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:43:15.524 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=started 2024-09-14T08:43:15.524 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:43:15.524 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:43:15.524 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:43:16.524 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_replay_state cluster1 mirror test started 2024-09-14T08:43:16.525 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:43:16.525 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:43:16.525 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:43:16.525 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=started 2024-09-14T08:43:16.525 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_result 2024-09-14T08:43:16.525 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=stopped 2024-09-14T08:43:16.526 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2024-09-14T08:43:16.526 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_instance=cluster1 2024-09-14T08:43:16.526 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:43:16.526 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ grep -i state 2024-09-14T08:43:16.526 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T08:43:16.526 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance=cluster1 2024-09-14T08:43:16.527 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local loop_instance 2024-09-14T08:43:16.527 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:43:16.527 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sleep 0 2024-09-14T08:43:16.527 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T08:43:16.527 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ seq 0 3 2024-09-14T08:43:16.529 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:43:16.529 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2024-09-14T08:43:16.529 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:0 2024-09-14T08:43:16.529 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:43:16.530 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:43:16.530 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T08:43:16.530 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:0 2024-09-14T08:43:16.530 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:43:16.530 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:43:16.530 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:43:16.530 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=0 2024-09-14T08:43:16.530 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T08:43:16.530 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:43:16.530 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:43:16.530 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=0 2024-09-14T08:43:16.530 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:43:16.530 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T08:43:16.530 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:0 2024-09-14T08:43:16.530 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:43:16.531 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:43:16.531 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T08:43:16.531 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:0 2024-09-14T08:43:16.531 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:43:16.532 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:43:16.532 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:43:16.532 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:43:16.532 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T08:43:16.532 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:43:16.532 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:43:16.532 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=0 2024-09-14T08:43:16.532 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=0 2024-09-14T08:43:16.533 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T08:43:16.552 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:43:16.552 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:43:16.552 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:43:16.552 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2024-09-14T08:43:16.630 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T08:43:16.631 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:43:16.631 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:43:16.634 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:43:16.634 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2024-09-14T08:43:16.634 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:1 2024-09-14T08:43:16.634 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:43:16.634 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:43:16.634 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T08:43:16.634 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:1 2024-09-14T08:43:16.634 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:43:16.635 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:43:16.635 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:43:16.635 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=1 2024-09-14T08:43:16.635 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T08:43:16.635 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:43:16.635 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:43:16.635 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=1 2024-09-14T08:43:16.635 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:43:16.635 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T08:43:16.635 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:1 2024-09-14T08:43:16.635 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:43:16.635 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:43:16.635 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T08:43:16.635 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:1 2024-09-14T08:43:16.635 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:43:16.635 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:43:16.636 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:43:16.636 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:43:16.636 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T08:43:16.636 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:43:16.636 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:43:16.636 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=1 2024-09-14T08:43:16.636 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=1 2024-09-14T08:43:16.636 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T08:43:16.654 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:43:16.654 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:43:16.654 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:43:16.654 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2024-09-14T08:43:16.727 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T08:43:16.727 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:43:16.727 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:43:16.730 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:43:16.730 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2024-09-14T08:43:16.731 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:2 2024-09-14T08:43:16.731 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:43:16.731 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:43:16.731 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-14T08:43:16.731 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:2 2024-09-14T08:43:16.731 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:43:16.731 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:43:16.731 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:43:16.731 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=2 2024-09-14T08:43:16.731 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T08:43:16.731 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:43:16.731 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:43:16.731 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=2 2024-09-14T08:43:16.732 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:43:16.732 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-14T08:43:16.732 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:2 2024-09-14T08:43:16.732 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:43:16.732 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:43:16.732 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T08:43:16.732 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:2 2024-09-14T08:43:16.732 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:43:16.732 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:43:16.732 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:43:16.732 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:43:16.732 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-14T08:43:16.732 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:43:16.732 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:43:16.733 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=2 2024-09-14T08:43:16.733 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=2 2024-09-14T08:43:16.733 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T08:43:16.749 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:43:16.749 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:43:16.749 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:43:16.749 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2024-09-14T08:43:16.829 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ return 0 2024-09-14T08:43:16.830 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T08:43:16.830 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ echo ' "state": "Replaying"' 2024-09-14T08:43:16.830 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep -i Replaying 2024-09-14T08:43:16.831 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: "state": "Replaying" 2024-09-14T08:43:16.832 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_state=started 2024-09-14T08:43:16.832 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test started = started 2024-09-14T08:43:16.832 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:43:16.832 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2024-09-14T08:43:16.832 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:43:16.833 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:43:16.833 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:43:16.833 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:43:16.833 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = journal ']' 2024-09-14T08:43:16.833 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = snapshot ']' 2024-09-14T08:43:16.833 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test 2024-09-14T08:43:16.833 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:43:16.833 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:43:16.833 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:43:16.833 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:43:16.833 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster2-mirror-test.status 2024-09-14T08:43:16.833 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster2-mirror-test.status 2024-09-14T08:43:16.833 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:43:16.834 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-test.status 2024-09-14T08:43:16.834 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster1-mirror-test.status 2024-09-14T08:43:16.834 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster1-mirror-test.status 2024-09-14T08:43:16.834 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:43:16.835 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_status_log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:43:16.835 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ mirror_image_snapshot cluster2 mirror test 2024-09-14T08:43:16.835 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:43:16.835 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:43:16.835 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:43:16.835 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2024-09-14T08:43:18.527 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Snapshot ID: 29 2024-09-14T08:43:18.538 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.status 2024-09-14T08:43:18.538 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:43:18.538 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:43:18.539 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:43:18.539 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-test.status 2024-09-14T08:43:18.539 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2024-09-14T08:43:18.539 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:43:18.614 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-14T08:43:18.616 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snapshot_id=29 2024-09-14T08:43:18.616 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:43:18.616 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:43:18.616 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.2 2024-09-14T08:43:18.818 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:43:18.818 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:43:18.819 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:43:18.819 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:43:18.819 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:43:18.819 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-09-14T08:43:18.819 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:43:18.870 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:43:18.870 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:43:18.872 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:43:18.872 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:43:18.872 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:43:18.872 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:43:18.872 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 29 = '' 2024-09-14T08:43:18.872 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:43:18.872 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.4 2024-09-14T08:43:19.273 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:43:19.273 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:43:19.273 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:43:19.274 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:43:19.274 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:43:19.274 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-09-14T08:43:19.274 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:43:19.324 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:43:19.324 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:43:19.326 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:43:19.326 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:43:19.326 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:43:19.326 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:43:19.326 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 29 = '' 2024-09-14T08:43:19.326 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:43:19.327 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.8 2024-09-14T08:43:20.128 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:43:20.128 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:43:20.128 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:43:20.128 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:43:20.128 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:43:20.128 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-09-14T08:43:20.128 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:43:20.186 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:43:20.186 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:43:20.187 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:43:20.187 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:43:20.187 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:43:20.188 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:43:20.188 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 29 = '' 2024-09-14T08:43:20.188 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:43:20.188 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1.6 2024-09-14T08:43:21.789 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:43:21.789 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:43:21.790 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:43:21.790 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:43:21.790 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:43:21.790 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-09-14T08:43:21.790 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:43:22.053 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:43:22.054 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=29 2024-09-14T08:43:22.055 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 29 = 29 2024-09-14T08:43:22.055 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:43:22.055 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snap_present cluster1 mirror test snap9 2024-09-14T08:43:22.055 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:43:22.055 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:43:22.055 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:43:22.055 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap9 2024-09-14T08:43:22.055 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:43:22.055 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T08:43:22.055 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:43:23.055 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 info mirror/test@snap9 2024-09-14T08:43:23.103 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'test': 2024-09-14T08:43:23.103 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 512 MiB in 128 objects 2024-09-14T08:43:23.103 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:43:23.103 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 18 2024-09-14T08:43:23.103 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 10ba6324d73b 2024-09-14T08:43:23.103 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.10ba6324d73b 2024-09-14T08:43:23.103 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:43:23.104 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering, non-primary 2024-09-14T08:43:23.104 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:43:23.104 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:43:23.104 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:35:14 2024 2024-09-14T08:43:23.104 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:41:40 2024 2024-09-14T08:43:23.104 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:35:14 2024 2024-09-14T08:43:23.104 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: protected: False 2024-09-14T08:43:23.104 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:43:23.104 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:43:23.104 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: 1159fdda-fb30-49ff-ba1c-60890cdd16a0 2024-09-14T08:43:23.104 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: false 2024-09-14T08:43:23.107 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:43:23.107 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n snap8-clean ']' 2024-09-14T08:43:23.107 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ compare_image_snaps mirror test snap8-clean 2024-09-14T08:43:23.107 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:43:23.107 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:43:23.108 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap8-clean 2024-09-14T08:43:23.108 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local ret=0 2024-09-14T08:43:23.108 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local rmt_export=/tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export 2024-09-14T08:43:23.108 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local loc_export=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:43:23.108 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:43:23.108 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror export test@snap8-clean /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export 2024-09-14T08:43:23.753 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T08:43:23.764 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 -p mirror export test@snap8-clean /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:43:24.577 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T08:43:24.585 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cmp /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:43:24.797 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:43:24.911 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:43:24.911 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ compare_image_snaps mirror test snap9 2024-09-14T08:43:24.912 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:43:24.912 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:43:24.912 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap9 2024-09-14T08:43:24.912 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local ret=0 2024-09-14T08:43:24.912 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local rmt_export=/tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export 2024-09-14T08:43:24.912 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local loc_export=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:43:24.912 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:43:24.912 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror export test@snap9 /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export 2024-09-14T08:43:25.551 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T08:43:25.559 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 -p mirror export test@snap9 /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:43:26.354 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T08:43:26.363 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cmp /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:43:26.576 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:43:26.691 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:43:26.691 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ clean_snap_name=snap9-clean 2024-09-14T08:43:26.691 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_snap cluster2 mirror test snap9-clean 2024-09-14T08:43:26.692 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:43:26.692 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:43:26.692 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:43:26.692 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap9-clean 2024-09-14T08:43:26.692 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap9-clean --debug-rbd=20 --debug-journaler=20 2024-09-14T08:43:27.472 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 10` 2024-09-14T08:43:27.472 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ stress_write_image cluster2 mirror test 2024-09-14T08:43:27.472 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:43:27.472 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:43:27.473 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:43:27.473 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T08:43:27.474 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local duration=8 2024-09-14T08:43:27.474 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set +e 2024-09-14T08:43:27.474 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ timeout 8s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2024-09-14T08:43:27.865 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:100 IOs, 86275072 bytes 2024-09-14T08:43:28.086 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:200 IOs, 145164288 bytes 2024-09-14T08:43:28.243 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:300 IOs, 192870400 bytes 2024-09-14T08:43:28.482 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:400 IOs, 249267200 bytes 2024-09-14T08:43:28.593 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:500 IOs, 266040320 bytes 2024-09-14T08:43:28.774 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:600 IOs, 320418816 bytes 2024-09-14T08:43:28.924 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:700 IOs, 361776128 bytes 2024-09-14T08:43:29.116 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:800 IOs, 412461056 bytes 2024-09-14T08:43:29.346 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:900 IOs, 480980992 bytes 2024-09-14T08:43:29.468 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1000 IOs, 506390528 bytes 2024-09-14T08:43:29.602 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1100 IOs, 553397248 bytes 2024-09-14T08:43:29.869 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1200 IOs, 629354496 bytes 2024-09-14T08:43:29.949 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1300 IOs, 633232384 bytes 2024-09-14T08:43:30.069 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1400 IOs, 656011264 bytes 2024-09-14T08:43:30.194 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1500 IOs, 678849536 bytes 2024-09-14T08:43:30.401 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1600 IOs, 760187904 bytes 2024-09-14T08:43:30.543 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1700 IOs, 770182144 bytes 2024-09-14T08:43:30.765 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1800 IOs, 860098560 bytes 2024-09-14T08:43:30.951 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1900 IOs, 893690880 bytes 2024-09-14T08:43:31.103 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2000 IOs, 930264064 bytes 2024-09-14T08:43:31.183 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2100 IOs, 935469056 bytes 2024-09-14T08:43:31.369 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2200 IOs, 990418944 bytes 2024-09-14T08:43:31.528 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2300 IOs, 1027138560 bytes 2024-09-14T08:43:31.679 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2400 IOs, 1062631424 bytes 2024-09-14T08:43:31.887 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2500 IOs, 1132262400 bytes 2024-09-14T08:43:32.041 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2600 IOs, 1163827200 bytes 2024-09-14T08:43:32.153 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2700 IOs, 1186830336 bytes 2024-09-14T08:43:32.337 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2800 IOs, 1235179520 bytes 2024-09-14T08:43:32.460 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2900 IOs, 1269533696 bytes 2024-09-14T08:43:32.708 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3000 IOs, 1342600192 bytes 2024-09-14T08:43:32.813 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3100 IOs, 1356136448 bytes 2024-09-14T08:43:33.082 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3200 IOs, 1446653952 bytes 2024-09-14T08:43:33.270 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3300 IOs, 1513798656 bytes 2024-09-14T08:43:33.484 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3400 IOs, 1559361536 bytes 2024-09-14T08:43:33.775 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3500 IOs, 1663304704 bytes 2024-09-14T08:43:33.981 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3600 IOs, 1711152128 bytes 2024-09-14T08:43:34.163 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3700 IOs, 1759831040 bytes 2024-09-14T08:43:34.240 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3800 IOs, 1760466944 bytes 2024-09-14T08:43:34.390 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3900 IOs, 1793357824 bytes 2024-09-14T08:43:34.471 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4000 IOs, 1796252672 bytes 2024-09-14T08:43:34.544 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4100 IOs, 1797578752 bytes 2024-09-14T08:43:34.754 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4200 IOs, 1859497984 bytes 2024-09-14T08:43:34.906 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4300 IOs, 1896100864 bytes 2024-09-14T08:43:35.067 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4400 IOs, 1940613120 bytes 2024-09-14T08:43:35.217 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4500 IOs, 1974023168 bytes 2024-09-14T08:43:35.365 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4600 IOs, 2009861120 bytes 2024-09-14T08:43:35.478 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ error_code=124 2024-09-14T08:43:35.479 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set -e 2024-09-14T08:43:35.479 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T08:43:35.479 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:43:35.479 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2024-09-14T08:43:35.479 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:43:35.479 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:43:35.479 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:43:35.479 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state_pattern=up+replaying 2024-09-14T08:43:35.479 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local description_pattern= 2024-09-14T08:43:35.479 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local service_pattern= 2024-09-14T08:43:35.480 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:43:35.480 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:43:36.480 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2024-09-14T08:43:36.480 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:43:36.480 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:43:36.480 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:43:36.480 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state_pattern=up+replaying 2024-09-14T08:43:36.481 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local description_pattern= 2024-09-14T08:43:36.481 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local service_pattern= 2024-09-14T08:43:36.481 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status 2024-09-14T08:43:36.481 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ CEPH_ARGS= 2024-09-14T08:43:36.481 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2024-09-14T08:43:36.540 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ status='test: 2024-09-14T08:43:36.540 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: global_id: 1159fdda-fb30-49ff-ba1c-60890cdd16a0 2024-09-14T08:43:36.540 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: state: up+replaying 2024-09-14T08:43:36.540 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":531819008.0,"last_snapshot_bytes":533337088,"last_snapshot_sync_seconds":1,"local_snapshot_timestamp":1726303344,"remote_snapshot_timestamp":1726303344,"replay_state":"idle"} 2024-09-14T08:43:36.540 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: service: mirror.2 on smithi164 2024-09-14T08:43:36.541 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: last_update: 2024-09-14 08:43:12 2024-09-14T08:43:36.541 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: peer_sites: 2024-09-14T08:43:36.541 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: name: cluster2 2024-09-14T08:43:36.541 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: state: up+stopped 2024-09-14T08:43:36.541 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: description: local image is primary 2024-09-14T08:43:36.541 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: last_update: 2024-09-14 08:43:29' 2024-09-14T08:43:36.541 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep '^ state: .*up+replaying' 2024-09-14T08:43:36.542 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: state: up+replaying 2024-09-14T08:43:36.542 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep '^ description: .*' 2024-09-14T08:43:36.542 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":531819008.0,"last_snapshot_bytes":533337088,"last_snapshot_sync_seconds":1,"local_snapshot_timestamp":1726303344,"remote_snapshot_timestamp":1726303344,"replay_state":"idle"} 2024-09-14T08:43:36.543 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n '' ']' 2024-09-14T08:43:36.543 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ echo up+replaying 2024-09-14T08:43:36.543 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep '^up+' 2024-09-14T08:43:36.543 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:up+replaying 2024-09-14T08:43:36.544 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep 'service: *mirror..* on ' 2024-09-14T08:43:36.544 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: service: mirror.2 on smithi164 2024-09-14T08:43:36.545 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local last_update 2024-09-14T08:43:36.545 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-14T08:43:36.547 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ last_update='2024-09-14 08:43:12' 2024-09-14T08:43:36.547 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2024-09-14 08:43:12' 2024-09-14T08:43:36.547 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:43:36.547 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:43:36.547 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:43:36.547 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state_pattern=up+replaying 2024-09-14T08:43:36.547 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local 'prev_last_update=2024-09-14 08:43:12' 2024-09-14T08:43:36.548 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status 2024-09-14T08:43:36.548 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ CEPH_ARGS= 2024-09-14T08:43:36.548 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-14T08:43:36.609 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ status='OKOKOK142174290mirror.1smithi16419.3.0-4775-g7f24e589trueOK42214285mirror.2smithi16419.3.0-4775-g7f24e589falseOK44874489mirror.3smithi16419.3.0-4775-g7f24e589falseOK45544557mirror.0smithi16419.3.0-4775-g7f24e589falseOKtest1159fdda-fb30-49ff-ba1c-60890cdd16a0up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":531819008.0,"last_snapshot_bytes":533337088,"last_snapshot_sync_seconds":1,"local_snapshot_timestamp":1726303344,"remote_snapshot_timestamp":1726303344,"replay_state":"idle"}42214285mirror.2smithi1642024-09-14 08:43:12cluster29ab74346-7215-4f9b-943c-26e4f257da03up+stoppedlocal image is primary2024-09-14 08:43:29' 2024-09-14T08:43:36.610 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local last_update state 2024-09-14T08:43:36.610 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2024-09-14T08:43:36.611 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ last_update='2024-09-14 08:43:12' 2024-09-14T08:43:36.611 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2024-09-14T08:43:36.614 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ state=up+replaying 2024-09-14T08:43:36.614 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ echo up+replaying 2024-09-14T08:43:36.614 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep up+replaying 2024-09-14T08:43:36.614 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:up+replaying 2024-09-14T08:43:36.615 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:43:36.615 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:43:36.616 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ snap_name=snap10 2024-09-14T08:43:36.616 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_snap cluster2 mirror test snap10 2024-09-14T08:43:36.616 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:43:36.616 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:43:36.616 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:43:36.616 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap10 2024-09-14T08:43:36.616 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap10 --debug-rbd=20 --debug-journaler=20 2024-09-14T08:43:38.118 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.8367403618993287) > weight (0.25) 2024-09-14T08:43:38.118 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.9693020155253903) > weight (0.25) 2024-09-14T08:43:38.118 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.9323744499047503) > weight (0.25) 2024-09-14T08:43:38.118 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.3 with skip (0.4401740316553341) > weight (0.25) 2024-09-14T08:43:38.118 INFO:tasks.rbd_mirror_thrash:waiting for 103 secs before thrashing 2024-09-14T08:44:05.518 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_started cluster1 mirror test 2024-09-14T08:44:05.518 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:44:05.518 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:44:05.518 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:44:05.518 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2024-09-14T08:44:05.518 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:44:05.519 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:44:05.519 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:44:05.519 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=started 2024-09-14T08:44:05.519 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:44:05.519 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:44:05.519 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:44:06.519 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_replay_state cluster1 mirror test started 2024-09-14T08:44:06.519 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:44:06.519 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:44:06.519 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:44:06.519 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=started 2024-09-14T08:44:06.519 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_result 2024-09-14T08:44:06.519 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=stopped 2024-09-14T08:44:06.520 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2024-09-14T08:44:06.520 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_instance=cluster1 2024-09-14T08:44:06.520 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:44:06.520 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ grep -i state 2024-09-14T08:44:06.520 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T08:44:06.520 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance=cluster1 2024-09-14T08:44:06.520 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local loop_instance 2024-09-14T08:44:06.521 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:44:06.521 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sleep 0 2024-09-14T08:44:06.521 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T08:44:06.522 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ seq 0 3 2024-09-14T08:44:06.522 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:44:06.523 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2024-09-14T08:44:06.523 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:0 2024-09-14T08:44:06.523 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:44:06.523 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:44:06.523 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T08:44:06.523 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:0 2024-09-14T08:44:06.523 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:44:06.523 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:44:06.523 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:44:06.523 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=0 2024-09-14T08:44:06.523 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T08:44:06.523 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:44:06.523 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:44:06.523 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=0 2024-09-14T08:44:06.523 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:44:06.524 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T08:44:06.524 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:0 2024-09-14T08:44:06.524 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:44:06.524 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:44:06.524 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T08:44:06.525 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:0 2024-09-14T08:44:06.525 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:44:06.525 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:44:06.525 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:44:06.525 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:44:06.525 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T08:44:06.525 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:44:06.525 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:44:06.526 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=0 2024-09-14T08:44:06.526 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=0 2024-09-14T08:44:06.526 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T08:44:06.545 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:44:06.545 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:44:06.546 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:44:06.546 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2024-09-14T08:44:06.624 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T08:44:06.625 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:44:06.625 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:44:06.628 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:44:06.628 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2024-09-14T08:44:06.628 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:1 2024-09-14T08:44:06.628 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:44:06.629 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:44:06.629 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T08:44:06.629 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:1 2024-09-14T08:44:06.629 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:44:06.629 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:44:06.629 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:44:06.629 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=1 2024-09-14T08:44:06.629 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T08:44:06.629 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:44:06.629 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:44:06.629 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=1 2024-09-14T08:44:06.629 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:44:06.630 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T08:44:06.630 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:1 2024-09-14T08:44:06.630 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:44:06.630 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:44:06.630 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T08:44:06.630 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:1 2024-09-14T08:44:06.630 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:44:06.630 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:44:06.630 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:44:06.630 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:44:06.630 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T08:44:06.630 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:44:06.630 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:44:06.630 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=1 2024-09-14T08:44:06.630 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=1 2024-09-14T08:44:06.631 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T08:44:06.647 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:44:06.647 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:44:06.647 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:44:06.647 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2024-09-14T08:44:06.720 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T08:44:06.721 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:44:06.721 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:44:06.723 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:44:06.724 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2024-09-14T08:44:06.724 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:2 2024-09-14T08:44:06.724 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:44:06.724 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:44:06.724 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-14T08:44:06.724 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:2 2024-09-14T08:44:06.724 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:44:06.724 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:44:06.724 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:44:06.724 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=2 2024-09-14T08:44:06.724 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T08:44:06.724 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:44:06.724 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:44:06.725 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=2 2024-09-14T08:44:06.725 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:44:06.725 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-14T08:44:06.725 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:2 2024-09-14T08:44:06.725 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:44:06.725 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:44:06.725 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T08:44:06.725 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:2 2024-09-14T08:44:06.725 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:44:06.725 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:44:06.725 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:44:06.725 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:44:06.725 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-14T08:44:06.725 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:44:06.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:44:06.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=2 2024-09-14T08:44:06.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=2 2024-09-14T08:44:06.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T08:44:06.742 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:44:06.742 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:44:06.742 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:44:06.742 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2024-09-14T08:44:06.823 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ return 0 2024-09-14T08:44:06.824 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T08:44:06.824 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ echo ' "state": "Replaying"' 2024-09-14T08:44:06.824 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep -i Replaying 2024-09-14T08:44:06.824 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: "state": "Replaying" 2024-09-14T08:44:06.826 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_state=started 2024-09-14T08:44:06.826 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test started = started 2024-09-14T08:44:06.826 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:44:06.826 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2024-09-14T08:44:06.826 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:44:06.826 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:44:06.827 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:44:06.827 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:44:06.827 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = journal ']' 2024-09-14T08:44:06.827 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = snapshot ']' 2024-09-14T08:44:06.827 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test 2024-09-14T08:44:06.827 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:44:06.827 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:44:06.827 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:44:06.827 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:44:06.827 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster2-mirror-test.status 2024-09-14T08:44:06.827 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster2-mirror-test.status 2024-09-14T08:44:06.827 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:44:06.828 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-test.status 2024-09-14T08:44:06.828 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster1-mirror-test.status 2024-09-14T08:44:06.828 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster1-mirror-test.status 2024-09-14T08:44:06.828 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:44:06.829 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_status_log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:44:06.829 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ mirror_image_snapshot cluster2 mirror test 2024-09-14T08:44:06.829 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:44:06.829 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:44:06.829 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:44:06.829 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2024-09-14T08:44:08.532 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Snapshot ID: 32 2024-09-14T08:44:08.545 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.status 2024-09-14T08:44:08.545 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:44:08.545 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:44:08.545 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:44:08.545 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-test.status 2024-09-14T08:44:08.546 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2024-09-14T08:44:08.546 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:44:08.623 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-14T08:44:08.624 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snapshot_id=32 2024-09-14T08:44:08.625 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:44:08.625 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:44:08.625 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.2 2024-09-14T08:44:08.827 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:44:08.827 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:44:08.827 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:44:08.827 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:44:08.827 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:44:08.827 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-09-14T08:44:08.827 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:44:08.878 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:44:08.878 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:44:08.879 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:44:08.880 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:44:08.880 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:44:08.880 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:44:08.880 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 32 = '' 2024-09-14T08:44:08.880 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:44:08.880 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.4 2024-09-14T08:44:09.281 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:44:09.281 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:44:09.281 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:44:09.281 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:44:09.281 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:44:09.281 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-09-14T08:44:09.281 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:44:09.335 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:44:09.335 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:44:09.337 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:44:09.337 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:44:09.337 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:44:09.337 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:44:09.338 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 32 = '' 2024-09-14T08:44:09.338 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:44:09.338 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.8 2024-09-14T08:44:10.139 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:44:10.139 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:44:10.139 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:44:10.139 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:44:10.139 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:44:10.139 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-09-14T08:44:10.139 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:44:10.197 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:44:10.197 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:44:10.199 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:44:10.199 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:44:10.199 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:44:10.200 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:44:10.200 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 32 = '' 2024-09-14T08:44:10.203 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:44:10.203 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1.6 2024-09-14T08:44:11.800 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:44:11.800 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:44:11.801 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:44:11.801 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:44:11.801 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:44:11.801 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-09-14T08:44:11.801 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:44:11.892 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:44:11.892 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:44:11.894 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:44:11.894 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:44:11.894 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:44:11.894 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:44:11.894 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 32 = '' 2024-09-14T08:44:11.894 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:44:11.894 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 2 2024-09-14T08:44:13.895 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:44:13.896 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:44:13.896 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:44:13.896 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:44:13.896 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:44:13.896 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-09-14T08:44:13.896 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:44:13.954 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:44:13.956 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=32 2024-09-14T08:44:13.956 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 32 = 32 2024-09-14T08:44:13.956 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:44:13.957 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snap_present cluster1 mirror test snap10 2024-09-14T08:44:13.957 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:44:13.957 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:44:13.957 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:44:13.957 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap10 2024-09-14T08:44:13.957 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:44:13.957 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T08:44:13.957 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:44:14.957 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 info mirror/test@snap10 2024-09-14T08:44:15.011 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'test': 2024-09-14T08:44:15.011 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 512 MiB in 128 objects 2024-09-14T08:44:15.012 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:44:15.012 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 20 2024-09-14T08:44:15.012 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 10ba6324d73b 2024-09-14T08:44:15.012 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.10ba6324d73b 2024-09-14T08:44:15.012 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:44:15.012 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering, non-primary 2024-09-14T08:44:15.012 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:44:15.012 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:44:15.012 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:35:14 2024 2024-09-14T08:44:15.012 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:43:23 2024 2024-09-14T08:44:15.012 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:35:14 2024 2024-09-14T08:44:15.012 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: protected: False 2024-09-14T08:44:15.012 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:44:15.013 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:44:15.013 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: 1159fdda-fb30-49ff-ba1c-60890cdd16a0 2024-09-14T08:44:15.013 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: false 2024-09-14T08:44:15.016 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:44:15.016 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n snap9-clean ']' 2024-09-14T08:44:15.016 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ compare_image_snaps mirror test snap9-clean 2024-09-14T08:44:15.016 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:44:15.016 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:44:15.016 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap9-clean 2024-09-14T08:44:15.016 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local ret=0 2024-09-14T08:44:15.016 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local rmt_export=/tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export 2024-09-14T08:44:15.016 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local loc_export=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:15.016 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:15.017 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror export test@snap9-clean /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export 2024-09-14T08:44:15.649 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T08:44:15.658 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 -p mirror export test@snap9-clean /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:16.376 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T08:44:16.384 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cmp /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:16.595 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:16.709 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:44:16.709 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ compare_image_snaps mirror test snap10 2024-09-14T08:44:16.709 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:44:16.710 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:44:16.710 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap10 2024-09-14T08:44:16.710 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local ret=0 2024-09-14T08:44:16.710 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local rmt_export=/tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export 2024-09-14T08:44:16.710 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local loc_export=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:16.710 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:16.710 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror export test@snap10 /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export 2024-09-14T08:44:17.345 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T08:44:17.357 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 -p mirror export test@snap10 /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:18.168 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T08:44:18.178 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cmp /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:18.388 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:18.506 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:44:18.506 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ clean_snap_name=snap10-clean 2024-09-14T08:44:18.507 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_snap cluster2 mirror test snap10-clean 2024-09-14T08:44:18.507 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:44:18.507 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:44:18.507 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:44:18.507 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap10-clean 2024-09-14T08:44:18.507 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap10-clean --debug-rbd=20 --debug-journaler=20 2024-09-14T08:44:19.485 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_started cluster1 mirror test 2024-09-14T08:44:19.485 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:44:19.485 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:44:19.485 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:44:19.485 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2024-09-14T08:44:19.485 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:44:19.485 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:44:19.485 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:44:19.485 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=started 2024-09-14T08:44:19.485 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:44:19.485 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:44:19.486 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:44:20.486 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_replay_state cluster1 mirror test started 2024-09-14T08:44:20.486 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:44:20.486 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:44:20.486 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:44:20.486 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=started 2024-09-14T08:44:20.486 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_result 2024-09-14T08:44:20.486 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=stopped 2024-09-14T08:44:20.487 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2024-09-14T08:44:20.488 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_instance=cluster1 2024-09-14T08:44:20.488 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:44:20.488 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ grep -i state 2024-09-14T08:44:20.488 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T08:44:20.488 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance=cluster1 2024-09-14T08:44:20.488 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local loop_instance 2024-09-14T08:44:20.488 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:44:20.488 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sleep 0 2024-09-14T08:44:20.488 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T08:44:20.488 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ seq 0 3 2024-09-14T08:44:20.491 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:44:20.491 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2024-09-14T08:44:20.491 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:0 2024-09-14T08:44:20.492 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:44:20.492 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:44:20.492 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T08:44:20.492 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:0 2024-09-14T08:44:20.492 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:44:20.492 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:44:20.492 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:44:20.492 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=0 2024-09-14T08:44:20.492 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T08:44:20.492 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:44:20.492 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:44:20.492 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=0 2024-09-14T08:44:20.492 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:44:20.492 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T08:44:20.493 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:0 2024-09-14T08:44:20.493 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:44:20.493 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:44:20.493 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T08:44:20.493 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:0 2024-09-14T08:44:20.493 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:44:20.493 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:44:20.493 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:44:20.493 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:44:20.493 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T08:44:20.493 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:44:20.494 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:44:20.494 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=0 2024-09-14T08:44:20.494 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=0 2024-09-14T08:44:20.494 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T08:44:20.512 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:44:20.513 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:44:20.513 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:44:20.513 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2024-09-14T08:44:20.591 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T08:44:20.592 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:44:20.592 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:44:20.593 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:44:20.593 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2024-09-14T08:44:20.593 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:1 2024-09-14T08:44:20.594 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:44:20.594 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:44:20.594 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T08:44:20.594 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:1 2024-09-14T08:44:20.594 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:44:20.594 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:44:20.594 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:44:20.594 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=1 2024-09-14T08:44:20.594 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T08:44:20.596 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:44:20.596 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:44:20.596 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=1 2024-09-14T08:44:20.596 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:44:20.596 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T08:44:20.596 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:1 2024-09-14T08:44:20.596 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:44:20.596 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:44:20.596 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T08:44:20.597 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:1 2024-09-14T08:44:20.597 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:44:20.597 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:44:20.597 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:44:20.597 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:44:20.597 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T08:44:20.597 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:44:20.597 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:44:20.597 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=1 2024-09-14T08:44:20.597 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=1 2024-09-14T08:44:20.597 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T08:44:20.615 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:44:20.615 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:44:20.615 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:44:20.615 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2024-09-14T08:44:20.689 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T08:44:20.689 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:44:20.689 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:44:20.691 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:44:20.691 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2024-09-14T08:44:20.691 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:2 2024-09-14T08:44:20.691 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:44:20.691 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:44:20.691 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-14T08:44:20.691 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:2 2024-09-14T08:44:20.691 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:44:20.691 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:44:20.692 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:44:20.692 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=2 2024-09-14T08:44:20.692 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T08:44:20.692 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:44:20.692 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:44:20.692 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=2 2024-09-14T08:44:20.692 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:44:20.694 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-14T08:44:20.694 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:2 2024-09-14T08:44:20.694 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:44:20.694 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:44:20.694 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T08:44:20.694 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:2 2024-09-14T08:44:20.694 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:44:20.694 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:44:20.694 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:44:20.694 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:44:20.694 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-14T08:44:20.694 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:44:20.694 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:44:20.694 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=2 2024-09-14T08:44:20.695 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=2 2024-09-14T08:44:20.695 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T08:44:20.712 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:44:20.712 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:44:20.712 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:44:20.712 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2024-09-14T08:44:20.792 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ return 0 2024-09-14T08:44:20.793 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T08:44:20.793 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ echo ' "state": "Replaying"' 2024-09-14T08:44:20.793 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep -i Replaying 2024-09-14T08:44:20.794 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: "state": "Replaying" 2024-09-14T08:44:20.795 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_state=started 2024-09-14T08:44:20.795 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test started = started 2024-09-14T08:44:20.795 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:44:20.795 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2024-09-14T08:44:20.796 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:44:20.796 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:44:20.796 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:44:20.796 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:44:20.796 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = journal ']' 2024-09-14T08:44:20.796 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = snapshot ']' 2024-09-14T08:44:20.796 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test 2024-09-14T08:44:20.796 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:44:20.796 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:44:20.796 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:44:20.796 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:44:20.796 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster2-mirror-test.status 2024-09-14T08:44:20.796 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster2-mirror-test.status 2024-09-14T08:44:20.796 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:44:20.797 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-test.status 2024-09-14T08:44:20.798 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster1-mirror-test.status 2024-09-14T08:44:20.798 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster1-mirror-test.status 2024-09-14T08:44:20.798 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:44:20.798 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_status_log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:44:20.798 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ mirror_image_snapshot cluster2 mirror test 2024-09-14T08:44:20.798 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:44:20.798 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:44:20.798 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:44:20.798 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2024-09-14T08:44:22.492 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Snapshot ID: 34 2024-09-14T08:44:22.504 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.status 2024-09-14T08:44:22.504 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:44:22.504 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:44:22.504 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:44:22.504 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-test.status 2024-09-14T08:44:22.505 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2024-09-14T08:44:22.505 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:44:22.584 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-14T08:44:22.586 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snapshot_id=34 2024-09-14T08:44:22.586 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:44:22.587 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:44:22.587 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.2 2024-09-14T08:44:22.788 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:44:22.788 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:44:22.788 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:44:22.788 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:44:22.788 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:44:22.788 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-09-14T08:44:22.789 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:44:22.843 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:44:22.844 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:44:22.845 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:44:22.845 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:44:22.845 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:44:22.845 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:44:22.845 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 34 = '' 2024-09-14T08:44:22.845 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:44:22.845 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.4 2024-09-14T08:44:23.246 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:44:23.246 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:44:23.246 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:44:23.246 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:44:23.247 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:44:23.247 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-09-14T08:44:23.247 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:44:23.300 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:44:23.300 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:44:23.302 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:44:23.302 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:44:23.302 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:44:23.302 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:44:23.302 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 34 = '' 2024-09-14T08:44:23.302 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:44:23.302 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.8 2024-09-14T08:44:24.103 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:44:24.103 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:44:24.103 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:44:24.103 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:44:24.104 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.status 2024-09-14T08:44:24.104 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-09-14T08:44:24.104 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:44:24.162 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:44:24.164 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=34 2024-09-14T08:44:24.164 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 34 = 34 2024-09-14T08:44:24.164 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:44:24.164 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snap_present cluster1 mirror test snap10-clean 2024-09-14T08:44:24.164 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:44:24.164 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:44:24.164 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:44:24.165 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap10-clean 2024-09-14T08:44:24.165 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:44:24.165 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T08:44:24.165 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:44:25.165 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 info mirror/test@snap10-clean 2024-09-14T08:44:25.219 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'test': 2024-09-14T08:44:25.219 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 512 MiB in 128 objects 2024-09-14T08:44:25.219 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:44:25.219 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 21 2024-09-14T08:44:25.219 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 10ba6324d73b 2024-09-14T08:44:25.220 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.10ba6324d73b 2024-09-14T08:44:25.220 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:44:25.220 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering, non-primary 2024-09-14T08:44:25.220 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:44:25.220 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:44:25.220 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:35:14 2024 2024-09-14T08:44:25.220 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:43:23 2024 2024-09-14T08:44:25.220 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:35:14 2024 2024-09-14T08:44:25.220 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: protected: False 2024-09-14T08:44:25.220 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:44:25.220 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:44:25.220 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: 1159fdda-fb30-49ff-ba1c-60890cdd16a0 2024-09-14T08:44:25.220 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: false 2024-09-14T08:44:25.222 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:44:25.223 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ seq 1 10 2024-09-14T08:44:25.225 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 10` 2024-09-14T08:44:25.225 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ snap_name=snap1 2024-09-14T08:44:25.225 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ compare_image_snaps mirror test snap1 2024-09-14T08:44:25.225 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:44:25.225 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:44:25.225 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap1 2024-09-14T08:44:25.225 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local ret=0 2024-09-14T08:44:25.225 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local rmt_export=/tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export 2024-09-14T08:44:25.225 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local loc_export=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:25.225 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:25.226 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror export test@snap1 /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export 2024-09-14T08:44:25.862 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T08:44:25.872 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 -p mirror export test@snap1 /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:26.646 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T08:44:26.656 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cmp /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:26.866 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:26.986 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:44:26.986 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ snap_name=snap1-clean 2024-09-14T08:44:26.986 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ compare_image_snaps mirror test snap1-clean 2024-09-14T08:44:26.986 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:44:26.986 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:44:26.986 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap1-clean 2024-09-14T08:44:26.986 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local ret=0 2024-09-14T08:44:26.986 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local rmt_export=/tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export 2024-09-14T08:44:26.986 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local loc_export=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:26.986 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:26.987 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror export test@snap1-clean /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export 2024-09-14T08:44:27.609 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T08:44:27.619 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 -p mirror export test@snap1-clean /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:28.403 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T08:44:28.411 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cmp /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:28.623 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:28.740 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:44:28.740 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 10` 2024-09-14T08:44:28.740 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ snap_name=snap2 2024-09-14T08:44:28.740 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ compare_image_snaps mirror test snap2 2024-09-14T08:44:28.740 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:44:28.740 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:44:28.740 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap2 2024-09-14T08:44:28.741 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local ret=0 2024-09-14T08:44:28.741 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local rmt_export=/tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export 2024-09-14T08:44:28.741 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local loc_export=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:28.741 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:28.741 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror export test@snap2 /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export 2024-09-14T08:44:29.394 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T08:44:29.404 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 -p mirror export test@snap2 /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:30.168 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T08:44:30.177 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cmp /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:30.388 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:30.511 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:44:30.511 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ snap_name=snap2-clean 2024-09-14T08:44:30.511 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ compare_image_snaps mirror test snap2-clean 2024-09-14T08:44:30.512 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:44:30.512 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:44:30.512 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap2-clean 2024-09-14T08:44:30.512 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local ret=0 2024-09-14T08:44:30.512 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local rmt_export=/tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export 2024-09-14T08:44:30.512 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local loc_export=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:30.512 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:30.512 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror export test@snap2-clean /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export 2024-09-14T08:44:31.144 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T08:44:31.153 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 -p mirror export test@snap2-clean /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:31.905 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T08:44:31.913 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cmp /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:32.126 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:32.243 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:44:32.243 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 10` 2024-09-14T08:44:32.243 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ snap_name=snap3 2024-09-14T08:44:32.243 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ compare_image_snaps mirror test snap3 2024-09-14T08:44:32.243 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:44:32.244 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:44:32.244 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap3 2024-09-14T08:44:32.244 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local ret=0 2024-09-14T08:44:32.244 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local rmt_export=/tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export 2024-09-14T08:44:32.244 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local loc_export=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:32.244 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:32.244 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror export test@snap3 /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export 2024-09-14T08:44:32.857 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T08:44:32.867 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 -p mirror export test@snap3 /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:33.618 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T08:44:33.626 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cmp /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:33.836 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:33.953 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:44:33.954 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ snap_name=snap3-clean 2024-09-14T08:44:33.954 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ compare_image_snaps mirror test snap3-clean 2024-09-14T08:44:33.954 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:44:33.954 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:44:33.954 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap3-clean 2024-09-14T08:44:33.954 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local ret=0 2024-09-14T08:44:33.954 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local rmt_export=/tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export 2024-09-14T08:44:33.954 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local loc_export=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:33.954 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:33.954 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror export test@snap3-clean /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export 2024-09-14T08:44:34.565 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T08:44:34.576 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 -p mirror export test@snap3-clean /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:35.380 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T08:44:35.387 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cmp /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:35.600 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:35.720 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:44:35.720 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 10` 2024-09-14T08:44:35.720 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ snap_name=snap4 2024-09-14T08:44:35.720 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ compare_image_snaps mirror test snap4 2024-09-14T08:44:35.720 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:44:35.720 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:44:35.720 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap4 2024-09-14T08:44:35.720 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local ret=0 2024-09-14T08:44:35.720 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local rmt_export=/tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export 2024-09-14T08:44:35.721 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local loc_export=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:35.721 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:35.721 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror export test@snap4 /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export 2024-09-14T08:44:36.310 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T08:44:36.323 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 -p mirror export test@snap4 /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:37.110 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T08:44:37.118 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cmp /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:37.331 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:37.449 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:44:37.449 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ snap_name=snap4-clean 2024-09-14T08:44:37.449 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ compare_image_snaps mirror test snap4-clean 2024-09-14T08:44:37.449 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:44:37.449 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:44:37.449 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap4-clean 2024-09-14T08:44:37.449 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local ret=0 2024-09-14T08:44:37.449 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local rmt_export=/tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export 2024-09-14T08:44:37.449 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local loc_export=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:37.449 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:37.450 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror export test@snap4-clean /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export 2024-09-14T08:44:38.074 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T08:44:38.085 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 -p mirror export test@snap4-clean /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:38.904 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.37148683196571564) > weight (0.25) 2024-09-14T08:44:38.904 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.4167901652648608) > weight (0.25) 2024-09-14T08:44:38.904 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.445352529937006) > weight (0.25) 2024-09-14T08:44:38.904 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.3 with skip (0.706628502417405) > weight (0.25) 2024-09-14T08:44:38.904 INFO:tasks.rbd_mirror_thrash:waiting for 79 secs before thrashing 2024-09-14T08:44:38.914 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T08:44:38.925 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cmp /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:39.136 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:39.253 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:44:39.253 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 10` 2024-09-14T08:44:39.253 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ snap_name=snap5 2024-09-14T08:44:39.253 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ compare_image_snaps mirror test snap5 2024-09-14T08:44:39.253 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:44:39.253 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:44:39.253 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap5 2024-09-14T08:44:39.253 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local ret=0 2024-09-14T08:44:39.254 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local rmt_export=/tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export 2024-09-14T08:44:39.254 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local loc_export=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:39.254 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:39.254 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror export test@snap5 /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export 2024-09-14T08:44:39.884 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T08:44:39.894 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 -p mirror export test@snap5 /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:40.561 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T08:44:40.568 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cmp /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:40.778 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:40.898 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:44:40.898 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ snap_name=snap5-clean 2024-09-14T08:44:40.898 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ compare_image_snaps mirror test snap5-clean 2024-09-14T08:44:40.898 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:44:40.898 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:44:40.898 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap5-clean 2024-09-14T08:44:40.898 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local ret=0 2024-09-14T08:44:40.898 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local rmt_export=/tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export 2024-09-14T08:44:40.898 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local loc_export=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:40.898 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:40.898 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror export test@snap5-clean /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export 2024-09-14T08:44:41.535 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T08:44:41.545 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 -p mirror export test@snap5-clean /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:42.254 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T08:44:42.262 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cmp /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:42.476 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:42.597 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:44:42.598 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 10` 2024-09-14T08:44:42.598 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ snap_name=snap6 2024-09-14T08:44:42.598 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ compare_image_snaps mirror test snap6 2024-09-14T08:44:42.598 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:44:42.598 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:44:42.598 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap6 2024-09-14T08:44:42.598 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local ret=0 2024-09-14T08:44:42.598 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local rmt_export=/tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export 2024-09-14T08:44:42.598 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local loc_export=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:42.598 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:42.598 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror export test@snap6 /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export 2024-09-14T08:44:43.252 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T08:44:43.262 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 -p mirror export test@snap6 /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:43.954 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T08:44:43.961 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cmp /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:44.173 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:44.291 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:44:44.291 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ snap_name=snap6-clean 2024-09-14T08:44:44.291 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ compare_image_snaps mirror test snap6-clean 2024-09-14T08:44:44.291 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:44:44.292 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:44:44.292 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap6-clean 2024-09-14T08:44:44.292 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local ret=0 2024-09-14T08:44:44.292 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local rmt_export=/tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export 2024-09-14T08:44:44.292 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local loc_export=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:44.292 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:44.292 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror export test@snap6-clean /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export 2024-09-14T08:44:44.917 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T08:44:44.928 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 -p mirror export test@snap6-clean /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:45.755 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T08:44:45.763 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cmp /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:45.976 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:46.095 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:44:46.095 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 10` 2024-09-14T08:44:46.095 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ snap_name=snap7 2024-09-14T08:44:46.096 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ compare_image_snaps mirror test snap7 2024-09-14T08:44:46.096 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:44:46.096 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:44:46.096 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap7 2024-09-14T08:44:46.096 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local ret=0 2024-09-14T08:44:46.096 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local rmt_export=/tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export 2024-09-14T08:44:46.096 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local loc_export=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:46.096 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:46.096 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror export test@snap7 /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export 2024-09-14T08:44:46.715 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T08:44:46.725 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 -p mirror export test@snap7 /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:47.599 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T08:44:47.610 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cmp /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:47.821 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:47.941 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:44:47.942 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ snap_name=snap7-clean 2024-09-14T08:44:47.942 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ compare_image_snaps mirror test snap7-clean 2024-09-14T08:44:47.942 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:44:47.942 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:44:47.942 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap7-clean 2024-09-14T08:44:47.942 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local ret=0 2024-09-14T08:44:47.942 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local rmt_export=/tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export 2024-09-14T08:44:47.942 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local loc_export=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:47.942 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:47.942 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror export test@snap7-clean /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export 2024-09-14T08:44:48.591 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T08:44:48.602 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 -p mirror export test@snap7-clean /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:49.366 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T08:44:49.376 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cmp /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:49.589 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:49.710 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:44:49.710 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 10` 2024-09-14T08:44:49.710 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ snap_name=snap8 2024-09-14T08:44:49.710 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ compare_image_snaps mirror test snap8 2024-09-14T08:44:49.710 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:44:49.710 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:44:49.710 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap8 2024-09-14T08:44:49.710 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local ret=0 2024-09-14T08:44:49.711 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local rmt_export=/tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export 2024-09-14T08:44:49.711 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local loc_export=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:49.711 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:49.711 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror export test@snap8 /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export 2024-09-14T08:44:50.348 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T08:44:50.359 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 -p mirror export test@snap8 /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:51.085 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T08:44:51.094 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cmp /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:51.306 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:51.424 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:44:51.424 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ snap_name=snap8-clean 2024-09-14T08:44:51.424 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ compare_image_snaps mirror test snap8-clean 2024-09-14T08:44:51.424 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:44:51.424 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:44:51.424 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap8-clean 2024-09-14T08:44:51.424 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local ret=0 2024-09-14T08:44:51.424 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local rmt_export=/tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export 2024-09-14T08:44:51.425 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local loc_export=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:51.425 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:51.425 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror export test@snap8-clean /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export 2024-09-14T08:44:52.064 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T08:44:52.074 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 -p mirror export test@snap8-clean /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:52.918 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T08:44:52.929 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cmp /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:53.140 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:53.261 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:44:53.262 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 10` 2024-09-14T08:44:53.262 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ snap_name=snap9 2024-09-14T08:44:53.262 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ compare_image_snaps mirror test snap9 2024-09-14T08:44:53.262 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:44:53.262 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:44:53.262 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap9 2024-09-14T08:44:53.262 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local ret=0 2024-09-14T08:44:53.262 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local rmt_export=/tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export 2024-09-14T08:44:53.262 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local loc_export=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:53.262 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:53.262 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror export test@snap9 /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export 2024-09-14T08:44:53.892 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T08:44:53.901 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 -p mirror export test@snap9 /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:54.704 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T08:44:54.713 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cmp /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:54.924 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:55.043 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:44:55.043 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ snap_name=snap9-clean 2024-09-14T08:44:55.044 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ compare_image_snaps mirror test snap9-clean 2024-09-14T08:44:55.044 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:44:55.044 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:44:55.044 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap9-clean 2024-09-14T08:44:55.044 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local ret=0 2024-09-14T08:44:55.044 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local rmt_export=/tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export 2024-09-14T08:44:55.044 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local loc_export=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:55.044 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:55.044 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror export test@snap9-clean /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export 2024-09-14T08:44:55.660 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T08:44:55.669 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 -p mirror export test@snap9-clean /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:56.425 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T08:44:56.435 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cmp /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:56.647 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:56.765 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:44:56.765 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 10` 2024-09-14T08:44:56.765 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ snap_name=snap10 2024-09-14T08:44:56.765 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ compare_image_snaps mirror test snap10 2024-09-14T08:44:56.766 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:44:56.766 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:44:56.766 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap10 2024-09-14T08:44:56.766 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local ret=0 2024-09-14T08:44:56.766 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local rmt_export=/tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export 2024-09-14T08:44:56.766 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local loc_export=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:56.766 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:56.766 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror export test@snap10 /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export 2024-09-14T08:44:57.385 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T08:44:57.396 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 -p mirror export test@snap10 /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:58.267 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T08:44:58.277 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cmp /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:58.489 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:58.609 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:44:58.610 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ snap_name=snap10-clean 2024-09-14T08:44:58.610 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ compare_image_snaps mirror test snap10-clean 2024-09-14T08:44:58.610 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:44:58.610 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:44:58.610 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap10-clean 2024-09-14T08:44:58.610 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local ret=0 2024-09-14T08:44:58.610 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local rmt_export=/tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export 2024-09-14T08:44:58.610 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local loc_export=/tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:58.610 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:44:58.610 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror export test@snap10-clean /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export 2024-09-14T08:44:59.263 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T08:44:59.274 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 -p mirror export test@snap10-clean /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:45:00.077 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T08:45:00.085 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cmp /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:45:00.298 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-test.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-test.export 2024-09-14T08:45:00.416 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:45:00.416 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ seq 1 10 2024-09-14T08:45:00.417 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 10` 2024-09-14T08:45:00.418 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ snap_name=snap1 2024-09-14T08:45:00.418 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_snapshot cluster2 mirror test snap1 2024-09-14T08:45:00.418 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:45:00.418 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:45:00.418 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:45:00.418 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap=snap1 2024-09-14T08:45:00.418 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap1 2024-09-14T08:45:00.525 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing snap: 100% complete...done. 2024-09-14T08:45:00.538 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ snap_name=snap1-clean 2024-09-14T08:45:00.538 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_snapshot cluster2 mirror test snap1-clean 2024-09-14T08:45:00.538 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:45:00.538 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:45:00.538 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:45:00.538 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap=snap1-clean 2024-09-14T08:45:00.538 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap1-clean 2024-09-14T08:45:01.536 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing snap: 100% complete...done. 2024-09-14T08:45:01.555 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 10` 2024-09-14T08:45:01.555 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ snap_name=snap2 2024-09-14T08:45:01.555 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_snapshot cluster2 mirror test snap2 2024-09-14T08:45:01.555 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:45:01.555 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:45:01.555 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:45:01.555 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap=snap2 2024-09-14T08:45:01.556 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap2 2024-09-14T08:45:02.536 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing snap: 100% complete...done. 2024-09-14T08:45:02.552 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ snap_name=snap2-clean 2024-09-14T08:45:02.552 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_snapshot cluster2 mirror test snap2-clean 2024-09-14T08:45:02.552 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:45:02.552 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:45:02.553 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:45:02.553 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap=snap2-clean 2024-09-14T08:45:02.553 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap2-clean 2024-09-14T08:45:03.552 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing snap: 100% complete...done. 2024-09-14T08:45:03.565 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 10` 2024-09-14T08:45:03.565 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ snap_name=snap3 2024-09-14T08:45:03.565 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_snapshot cluster2 mirror test snap3 2024-09-14T08:45:03.565 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:45:03.565 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:45:03.565 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:45:03.565 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap=snap3 2024-09-14T08:45:03.565 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap3 2024-09-14T08:45:04.548 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing snap: 100% complete...done. 2024-09-14T08:45:04.560 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ snap_name=snap3-clean 2024-09-14T08:45:04.560 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_snapshot cluster2 mirror test snap3-clean 2024-09-14T08:45:04.560 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:45:04.560 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:45:04.560 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:45:04.560 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap=snap3-clean 2024-09-14T08:45:04.560 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap3-clean 2024-09-14T08:45:05.568 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing snap: 100% complete...done. 2024-09-14T08:45:05.583 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 10` 2024-09-14T08:45:05.583 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ snap_name=snap4 2024-09-14T08:45:05.583 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_snapshot cluster2 mirror test snap4 2024-09-14T08:45:05.583 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:45:05.583 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:45:05.583 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:45:05.583 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap=snap4 2024-09-14T08:45:05.583 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap4 2024-09-14T08:45:06.561 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing snap: 100% complete...done. 2024-09-14T08:45:06.571 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ snap_name=snap4-clean 2024-09-14T08:45:06.571 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_snapshot cluster2 mirror test snap4-clean 2024-09-14T08:45:06.571 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:45:06.571 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:45:06.572 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:45:06.572 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap=snap4-clean 2024-09-14T08:45:06.572 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap4-clean 2024-09-14T08:45:07.574 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing snap: 100% complete...done. 2024-09-14T08:45:07.591 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 10` 2024-09-14T08:45:07.592 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ snap_name=snap5 2024-09-14T08:45:07.592 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_snapshot cluster2 mirror test snap5 2024-09-14T08:45:07.592 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:45:07.592 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:45:07.592 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:45:07.592 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap=snap5 2024-09-14T08:45:07.592 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap5 2024-09-14T08:45:08.574 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing snap: 100% complete...done. 2024-09-14T08:45:08.586 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ snap_name=snap5-clean 2024-09-14T08:45:08.586 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_snapshot cluster2 mirror test snap5-clean 2024-09-14T08:45:08.586 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:45:08.586 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:45:08.586 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:45:08.587 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap=snap5-clean 2024-09-14T08:45:08.587 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap5-clean 2024-09-14T08:45:09.582 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing snap: 100% complete...done. 2024-09-14T08:45:09.597 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 10` 2024-09-14T08:45:09.597 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ snap_name=snap6 2024-09-14T08:45:09.597 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_snapshot cluster2 mirror test snap6 2024-09-14T08:45:09.597 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:45:09.597 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:45:09.598 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:45:09.598 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap=snap6 2024-09-14T08:45:09.598 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap6 2024-09-14T08:45:10.485 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing snap: 100% complete...done. 2024-09-14T08:45:10.500 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ snap_name=snap6-clean 2024-09-14T08:45:10.500 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_snapshot cluster2 mirror test snap6-clean 2024-09-14T08:45:10.500 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:45:10.500 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:45:10.500 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:45:10.500 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap=snap6-clean 2024-09-14T08:45:10.500 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap6-clean 2024-09-14T08:45:11.586 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing snap: 100% complete...done. 2024-09-14T08:45:11.596 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 10` 2024-09-14T08:45:11.596 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ snap_name=snap7 2024-09-14T08:45:11.596 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_snapshot cluster2 mirror test snap7 2024-09-14T08:45:11.597 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:45:11.597 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:45:11.597 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:45:11.597 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap=snap7 2024-09-14T08:45:11.597 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap7 2024-09-14T08:45:12.586 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing snap: 100% complete...done. 2024-09-14T08:45:12.595 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ snap_name=snap7-clean 2024-09-14T08:45:12.595 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_snapshot cluster2 mirror test snap7-clean 2024-09-14T08:45:12.595 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:45:12.595 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:45:12.595 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:45:12.595 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap=snap7-clean 2024-09-14T08:45:12.595 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap7-clean 2024-09-14T08:45:13.603 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing snap: 100% complete...done. 2024-09-14T08:45:13.615 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 10` 2024-09-14T08:45:13.616 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ snap_name=snap8 2024-09-14T08:45:13.616 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_snapshot cluster2 mirror test snap8 2024-09-14T08:45:13.616 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:45:13.616 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:45:13.616 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:45:13.616 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap=snap8 2024-09-14T08:45:13.616 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap8 2024-09-14T08:45:14.596 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing snap: 100% complete...done. 2024-09-14T08:45:14.620 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ snap_name=snap8-clean 2024-09-14T08:45:14.620 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_snapshot cluster2 mirror test snap8-clean 2024-09-14T08:45:14.620 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:45:14.620 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:45:14.620 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:45:14.620 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap=snap8-clean 2024-09-14T08:45:14.620 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap8-clean 2024-09-14T08:45:15.606 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing snap: 100% complete...done. 2024-09-14T08:45:15.618 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 10` 2024-09-14T08:45:15.618 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ snap_name=snap9 2024-09-14T08:45:15.619 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_snapshot cluster2 mirror test snap9 2024-09-14T08:45:15.619 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:45:15.619 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:45:15.619 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:45:15.619 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap=snap9 2024-09-14T08:45:15.619 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap9 2024-09-14T08:45:16.610 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing snap: 100% complete...done. 2024-09-14T08:45:16.623 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ snap_name=snap9-clean 2024-09-14T08:45:16.623 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_snapshot cluster2 mirror test snap9-clean 2024-09-14T08:45:16.623 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:45:16.623 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:45:16.623 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:45:16.623 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap=snap9-clean 2024-09-14T08:45:16.624 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap9-clean 2024-09-14T08:45:17.621 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing snap: 100% complete...done. 2024-09-14T08:45:17.633 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 10` 2024-09-14T08:45:17.633 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ snap_name=snap10 2024-09-14T08:45:17.633 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_snapshot cluster2 mirror test snap10 2024-09-14T08:45:17.633 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:45:17.633 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:45:17.633 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:45:17.634 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap=snap10 2024-09-14T08:45:17.634 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap10 2024-09-14T08:45:18.609 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing snap: 100% complete...done. 2024-09-14T08:45:18.621 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ snap_name=snap10-clean 2024-09-14T08:45:18.621 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_snapshot cluster2 mirror test snap10-clean 2024-09-14T08:45:18.621 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:45:18.621 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:45:18.622 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:45:18.622 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap=snap10-clean 2024-09-14T08:45:18.622 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap10-clean 2024-09-14T08:45:19.615 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing snap: 100% complete...done. 2024-09-14T08:45:19.626 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image_retry cluster2 mirror test 2024-09-14T08:45:19.626 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:45:19.626 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:45:19.626 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:45:19.626 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T08:45:19.626 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0 2024-09-14T08:45:19.630 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror test 2024-09-14T08:45:19.630 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:45:19.631 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:45:19.631 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:45:19.631 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/test 2024-09-14T08:45:19.689 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/test 2024-09-14T08:45:19.783 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi164.stderr:2024-09-14T08:45:19.782+0000 7f3753e08640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55c94bd9c340 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-14T08:45:19.783 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi164.stderr:2024-09-14T08:45:19.782+0000 7f375ae16640 -1 rbd::mirror::ImageReplayer: 0x55c94ba4cc80 [3/1159fdda-fb30-49ff-ba1c-60890cdd16a0] operator(): start failed: (85) Interrupted system call should be restarted 2024-09-14T08:45:20.666 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-14T08:45:20.670 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:45:20.670 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_present cluster1 mirror test deleted 2024-09-14T08:45:20.670 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:45:20.670 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:45:20.670 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:45:20.670 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=deleted 2024-09-14T08:45:20.670 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T08:45:20.670 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:45:20.670 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T08:45:20.670 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror test 2024-09-14T08:45:20.670 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T08:45:20.670 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T08:45:20.670 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=test 2024-09-14T08:45:20.671 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/test 2024-09-14T08:45:20.671 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T08:45:20.705 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image test: (2) No such file or directory 2024-09-14T08:45:20.708 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image_id= 2024-09-14T08:45:20.708 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T08:45:20.708 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.1 2024-09-14T08:45:20.810 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_present cluster1 mirror test deleted '' 2024-09-14T08:45:20.810 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:45:20.810 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:45:20.810 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=test 2024-09-14T08:45:20.810 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=deleted 2024-09-14T08:45:20.810 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T08:45:20.810 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=deleted 2024-09-14T08:45:20.810 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_image_id 2024-09-14T08:45:20.810 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror test 2024-09-14T08:45:20.810 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T08:45:20.811 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T08:45:20.811 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=test 2024-09-14T08:45:20.811 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/test 2024-09-14T08:45:20.811 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T08:45:20.846 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image test: (2) No such file or directory 2024-09-14T08:45:20.849 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_image_id= 2024-09-14T08:45:20.849 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T08:45:20.849 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test deleted = deleted 2024-09-14T08:45:20.849 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:45:20.850 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ testlog 'TEST: create many images' 2024-09-14T08:45:20.850 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ tee -a /tmp/tmp.IUd9LESfOA/rbd-mirror.test.log 2024-09-14T08:45:20.850 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ date '+%F %T' 2024-09-14T08:45:20.850 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ echo 2024-09-14 08:45:20 TEST: create many images 2024-09-14T08:45:20.850 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:2024-09-14 08:45:20 TEST: create many images 2024-09-14T08:45:20.850 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ snap_name=snap 2024-09-14T08:45:20.851 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ seq 1 50 2024-09-14T08:45:20.853 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T08:45:20.853 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_1 2024-09-14T08:45:20.853 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image_and_enable_mirror cluster2 mirror image_1 snapshot 128M 2024-09-14T08:45:20.853 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:45:20.853 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:45:20.853 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:45:20.853 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:45:20.853 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_1 2024-09-14T08:45:20.853 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:45:20.853 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:45:20.853 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n snapshot ']' 2024-09-14T08:45:20.854 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:45:20.854 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image cluster2 mirror image_1 128M 2024-09-14T08:45:20.854 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:45:20.854 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:45:20.854 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:45:20.854 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:45:20.854 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_1 2024-09-14T08:45:20.854 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:45:20.854 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local size=128 2024-09-14T08:45:20.854 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n 128M ']' 2024-09-14T08:45:20.854 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ size=128M 2024-09-14T08:45:20.854 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:45:20.854 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_1 2024-09-14T08:45:20.899 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T08:45:20.899 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:45:20.900 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' image = image ']' 2024-09-14T08:45:20.900 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:45:20.900 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ enable_mirror cluster2 mirror image_1 snapshot 2024-09-14T08:45:20.900 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:45:20.900 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:45:20.900 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_1 2024-09-14T08:45:20.900 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:45:20.900 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_1 snapshot 2024-09-14T08:45:21.119 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.0 2024-09-14T08:45:21.120 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Sent signal 15 2024-09-14T08:45:21.120 INFO:tasks.rbd_mirror_thrash:waiting for 8 secs before reviving daemons 2024-09-14T08:45:21.121 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi164.stderr:2024-09-14T08:45:21.121+0000 7fe7c23d9640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 (PID: 42236) UID: 1000 2024-09-14T08:45:21.627 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Mirroring enabled 2024-09-14T08:45:21.632 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 info mirror/image_1 2024-09-14T08:45:21.670 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_1': 2024-09-14T08:45:21.671 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T08:45:21.671 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:45:21.671 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 1 2024-09-14T08:45:21.671 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 125c74df7b2c 2024-09-14T08:45:21.671 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.125c74df7b2c 2024-09-14T08:45:21.671 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:45:21.671 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering 2024-09-14T08:45:21.671 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:45:21.671 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:45:21.671 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:45:20 2024 2024-09-14T08:45:21.671 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:45:20 2024 2024-09-14T08:45:21.671 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:45:20 2024 2024-09-14T08:45:21.671 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:45:21.672 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:45:21.672 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: 06bb8003-76f2-47e8-a72f-feadf752d8d3 2024-09-14T08:45:21.672 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: true 2024-09-14T08:45:21.674 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n '' ']' 2024-09-14T08:45:21.674 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ stress_write_image cluster2 mirror image_1 2024-09-14T08:45:21.675 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:45:21.675 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:45:21.675 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_1 2024-09-14T08:45:21.675 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T08:45:21.676 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local duration=7 2024-09-14T08:45:21.676 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set +e 2024-09-14T08:45:21.676 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ timeout 7s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_1 --debug-rbd=20 --debug-journaler=20 2024-09-14T08:45:21.826 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:100 IOs, 27507712 bytes 2024-09-14T08:45:21.926 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:200 IOs, 56427520 bytes 2024-09-14T08:45:22.054 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:300 IOs, 66419712 bytes 2024-09-14T08:45:22.209 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:400 IOs, 111403008 bytes 2024-09-14T08:45:22.350 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:500 IOs, 141521920 bytes 2024-09-14T08:45:22.489 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:600 IOs, 179089408 bytes 2024-09-14T08:45:22.725 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:700 IOs, 249378816 bytes 2024-09-14T08:45:22.891 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:800 IOs, 294774784 bytes 2024-09-14T08:45:22.996 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:900 IOs, 321172480 bytes 2024-09-14T08:45:23.173 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1000 IOs, 373378048 bytes 2024-09-14T08:45:23.270 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1100 IOs, 373987328 bytes 2024-09-14T08:45:23.414 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1200 IOs, 410803200 bytes 2024-09-14T08:45:23.477 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1300 IOs, 411427840 bytes 2024-09-14T08:45:23.680 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1400 IOs, 480549888 bytes 2024-09-14T08:45:23.803 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1500 IOs, 514116608 bytes 2024-09-14T08:45:24.006 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1600 IOs, 559417344 bytes 2024-09-14T08:45:24.158 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1700 IOs, 598461440 bytes 2024-09-14T08:45:24.351 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1800 IOs, 652519424 bytes 2024-09-14T08:45:24.417 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1900 IOs, 655452160 bytes 2024-09-14T08:45:24.748 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2000 IOs, 770681856 bytes 2024-09-14T08:45:24.875 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2100 IOs, 799655936 bytes 2024-09-14T08:45:25.148 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2200 IOs, 889913344 bytes 2024-09-14T08:45:25.213 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2300 IOs, 906752000 bytes 2024-09-14T08:45:25.313 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2400 IOs, 907353088 bytes 2024-09-14T08:45:25.416 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2500 IOs, 923507712 bytes 2024-09-14T08:45:25.575 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2600 IOs, 965907456 bytes 2024-09-14T08:45:25.824 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2700 IOs, 1045790720 bytes 2024-09-14T08:45:26.075 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2800 IOs, 1140251648 bytes 2024-09-14T08:45:26.223 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2900 IOs, 1165295616 bytes 2024-09-14T08:45:26.412 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3000 IOs, 1218527232 bytes 2024-09-14T08:45:26.556 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3100 IOs, 1275615232 bytes 2024-09-14T08:45:26.757 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3200 IOs, 1318426624 bytes 2024-09-14T08:45:26.846 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3300 IOs, 1328746496 bytes 2024-09-14T08:45:26.989 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3400 IOs, 1365341184 bytes 2024-09-14T08:45:27.059 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3500 IOs, 1365941248 bytes 2024-09-14T08:45:27.131 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3600 IOs, 1370771456 bytes 2024-09-14T08:45:27.301 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3700 IOs, 1417580544 bytes 2024-09-14T08:45:27.481 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3800 IOs, 1479873536 bytes 2024-09-14T08:45:27.639 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3900 IOs, 1511401472 bytes 2024-09-14T08:45:27.800 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4000 IOs, 1564559360 bytes 2024-09-14T08:45:27.906 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4100 IOs, 1575620608 bytes 2024-09-14T08:45:28.077 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4200 IOs, 1624203264 bytes 2024-09-14T08:45:28.240 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4300 IOs, 1685756928 bytes 2024-09-14T08:45:28.376 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4400 IOs, 1696149504 bytes 2024-09-14T08:45:28.447 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4500 IOs, 1700640768 bytes 2024-09-14T08:45:28.565 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4600 IOs, 1722090496 bytes 2024-09-14T08:45:28.682 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ error_code=124 2024-09-14T08:45:28.683 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set -e 2024-09-14T08:45:28.683 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T08:45:28.683 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:45:28.683 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T08:45:28.683 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_2 2024-09-14T08:45:28.683 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image_and_enable_mirror cluster2 mirror image_2 snapshot 128M 2024-09-14T08:45:28.683 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:45:28.683 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:45:28.683 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:45:28.683 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:45:28.683 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_2 2024-09-14T08:45:28.683 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:45:28.683 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:45:28.684 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n snapshot ']' 2024-09-14T08:45:28.684 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:45:28.684 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image cluster2 mirror image_2 128M 2024-09-14T08:45:28.684 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:45:28.684 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:45:28.685 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:45:28.685 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:45:28.685 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_2 2024-09-14T08:45:28.685 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:45:28.685 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local size=128 2024-09-14T08:45:28.685 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n 128M ']' 2024-09-14T08:45:28.685 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ size=128M 2024-09-14T08:45:28.685 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:45:28.685 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_2 2024-09-14T08:45:28.729 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T08:45:28.729 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:45:28.729 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' image = image ']' 2024-09-14T08:45:28.729 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:45:28.730 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ enable_mirror cluster2 mirror image_2 snapshot 2024-09-14T08:45:28.730 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:45:28.730 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:45:28.730 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_2 2024-09-14T08:45:28.730 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:45:28.730 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_2 snapshot 2024-09-14T08:45:29.121 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.0 2024-09-14T08:45:29.121 INFO:teuthology.orchestra.run:waiting for 600 2024-09-14T08:45:29.122 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.0 2024-09-14T08:45:29.122 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Restarting daemon 2024-09-14T08:45:29.122 DEBUG:teuthology.orchestra.run.smithi164:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 2024-09-14T08:45:29.125 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Started 2024-09-14T08:45:29.125 INFO:tasks.rbd_mirror_thrash:waiting for 91 secs before thrashing 2024-09-14T08:45:29.495 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Mirroring enabled 2024-09-14T08:45:29.500 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 info mirror/image_2 2024-09-14T08:45:29.539 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_2': 2024-09-14T08:45:29.539 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T08:45:29.539 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:45:29.539 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 1 2024-09-14T08:45:29.540 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 1268d666b99d 2024-09-14T08:45:29.540 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.1268d666b99d 2024-09-14T08:45:29.540 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:45:29.540 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering 2024-09-14T08:45:29.540 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:45:29.540 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:45:29.540 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:45:28 2024 2024-09-14T08:45:29.540 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:45:28 2024 2024-09-14T08:45:29.540 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:45:28 2024 2024-09-14T08:45:29.540 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:45:29.540 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:45:29.540 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: f36140bd-57e9-420f-8e29-5672a8b58d3d 2024-09-14T08:45:29.540 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: true 2024-09-14T08:45:29.543 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n '' ']' 2024-09-14T08:45:29.543 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ stress_write_image cluster2 mirror image_2 2024-09-14T08:45:29.543 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:45:29.543 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:45:29.543 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_2 2024-09-14T08:45:29.543 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T08:45:29.545 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local duration=6 2024-09-14T08:45:29.545 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set +e 2024-09-14T08:45:29.545 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ timeout 6s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_2 --debug-rbd=20 --debug-journaler=20 2024-09-14T08:45:29.777 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:100 IOs, 61190144 bytes 2024-09-14T08:45:29.873 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:200 IOs, 72861696 bytes 2024-09-14T08:45:29.974 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:300 IOs, 101532672 bytes 2024-09-14T08:45:30.252 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:400 IOs, 186622976 bytes 2024-09-14T08:45:30.466 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:500 IOs, 249480192 bytes 2024-09-14T08:45:30.805 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:600 IOs, 381468672 bytes 2024-09-14T08:45:30.924 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:700 IOs, 396327936 bytes 2024-09-14T08:45:30.997 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:800 IOs, 396949504 bytes 2024-09-14T08:45:31.159 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:900 IOs, 451434496 bytes 2024-09-14T08:45:31.388 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1000 IOs, 510368768 bytes 2024-09-14T08:45:31.506 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1100 IOs, 540469248 bytes 2024-09-14T08:45:31.653 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1200 IOs, 566061056 bytes 2024-09-14T08:45:31.871 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1300 IOs, 639777792 bytes 2024-09-14T08:45:32.157 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1400 IOs, 730964992 bytes 2024-09-14T08:45:32.485 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1500 IOs, 807096320 bytes 2024-09-14T08:45:32.649 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1600 IOs, 859479040 bytes 2024-09-14T08:45:32.794 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1700 IOs, 901941248 bytes 2024-09-14T08:45:33.023 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1800 IOs, 958018560 bytes 2024-09-14T08:45:33.149 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1900 IOs, 988961792 bytes 2024-09-14T08:45:33.371 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2000 IOs, 1054103552 bytes 2024-09-14T08:45:33.632 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2100 IOs, 1142850560 bytes 2024-09-14T08:45:33.928 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2200 IOs, 1247397888 bytes 2024-09-14T08:45:34.223 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2300 IOs, 1347227648 bytes 2024-09-14T08:45:34.382 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2400 IOs, 1370807296 bytes 2024-09-14T08:45:34.535 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2500 IOs, 1419825152 bytes 2024-09-14T08:45:34.756 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2600 IOs, 1481232384 bytes 2024-09-14T08:45:34.909 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2700 IOs, 1512589312 bytes 2024-09-14T08:45:35.093 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2800 IOs, 1562767360 bytes 2024-09-14T08:45:35.355 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2900 IOs, 1646758912 bytes 2024-09-14T08:45:35.528 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3000 IOs, 1714526208 bytes 2024-09-14T08:45:35.551 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ error_code=124 2024-09-14T08:45:35.552 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set -e 2024-09-14T08:45:35.552 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T08:45:35.552 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:45:35.552 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T08:45:35.552 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_3 2024-09-14T08:45:35.552 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image_and_enable_mirror cluster2 mirror image_3 snapshot 128M 2024-09-14T08:45:35.552 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:45:35.552 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:45:35.552 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:45:35.552 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:45:35.552 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_3 2024-09-14T08:45:35.552 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:45:35.553 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:45:35.553 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n snapshot ']' 2024-09-14T08:45:35.553 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:45:35.553 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image cluster2 mirror image_3 128M 2024-09-14T08:45:35.553 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:45:35.553 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:45:35.553 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:45:35.553 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:45:35.553 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_3 2024-09-14T08:45:35.553 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:45:35.553 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local size=128 2024-09-14T08:45:35.553 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n 128M ']' 2024-09-14T08:45:35.553 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ size=128M 2024-09-14T08:45:35.553 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:45:35.554 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_3 2024-09-14T08:45:35.616 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T08:45:35.616 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:45:35.616 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' image = image ']' 2024-09-14T08:45:35.616 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:45:35.616 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ enable_mirror cluster2 mirror image_3 snapshot 2024-09-14T08:45:35.616 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:45:35.616 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:45:35.616 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_3 2024-09-14T08:45:35.617 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:45:35.617 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_3 snapshot 2024-09-14T08:45:36.510 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Mirroring enabled 2024-09-14T08:45:36.516 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 info mirror/image_3 2024-09-14T08:45:36.558 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_3': 2024-09-14T08:45:36.558 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T08:45:36.558 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:45:36.558 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 1 2024-09-14T08:45:36.558 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 127aeb5018d0 2024-09-14T08:45:36.558 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.127aeb5018d0 2024-09-14T08:45:36.559 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:45:36.559 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering 2024-09-14T08:45:36.559 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:45:36.559 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:45:36.559 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:45:35 2024 2024-09-14T08:45:36.559 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:45:35 2024 2024-09-14T08:45:36.559 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:45:35 2024 2024-09-14T08:45:36.559 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:45:36.559 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:45:36.559 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: c4fe1bd8-3565-46e8-87a1-574d1ec9a443 2024-09-14T08:45:36.559 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: true 2024-09-14T08:45:36.562 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n '' ']' 2024-09-14T08:45:36.562 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ stress_write_image cluster2 mirror image_3 2024-09-14T08:45:36.562 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:45:36.562 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:45:36.562 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_3 2024-09-14T08:45:36.562 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T08:45:36.564 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local duration=5 2024-09-14T08:45:36.564 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set +e 2024-09-14T08:45:36.564 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ timeout 5s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_3 --debug-rbd=20 --debug-journaler=20 2024-09-14T08:45:36.702 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:100 IOs, 14092288 bytes 2024-09-14T08:45:36.812 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:200 IOs, 33941504 bytes 2024-09-14T08:45:36.968 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:300 IOs, 80841728 bytes 2024-09-14T08:45:37.074 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:400 IOs, 101833728 bytes 2024-09-14T08:45:37.477 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:500 IOs, 158745600 bytes 2024-09-14T08:45:37.723 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:600 IOs, 236626944 bytes 2024-09-14T08:45:37.886 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:700 IOs, 285390848 bytes 2024-09-14T08:45:38.173 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:800 IOs, 370308096 bytes 2024-09-14T08:45:38.311 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:900 IOs, 421218304 bytes 2024-09-14T08:45:38.436 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1000 IOs, 431532032 bytes 2024-09-14T08:45:38.504 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1100 IOs, 439681024 bytes 2024-09-14T08:45:38.607 INFO:tasks.ceph.cluster2.osd.2.smithi164.stderr:2024-09-14T08:45:38.606+0000 7fe32301f640 -1 reset not still connected to 0x55e7574a5a00 2024-09-14T08:45:38.654 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1200 IOs, 482467840 bytes 2024-09-14T08:45:38.762 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1300 IOs, 490358784 bytes 2024-09-14T08:45:38.901 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1400 IOs, 531930112 bytes 2024-09-14T08:45:39.178 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1500 IOs, 619187200 bytes 2024-09-14T08:45:39.251 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1600 IOs, 619800576 bytes 2024-09-14T08:45:39.376 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1700 IOs, 644016128 bytes 2024-09-14T08:45:39.449 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1800 IOs, 644604928 bytes 2024-09-14T08:45:39.591 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1900 IOs, 678838272 bytes 2024-09-14T08:45:39.846 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2000 IOs, 761160704 bytes 2024-09-14T08:45:40.005 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2100 IOs, 804554752 bytes 2024-09-14T08:45:40.167 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2200 IOs, 843865088 bytes 2024-09-14T08:45:40.285 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2300 IOs, 868544512 bytes 2024-09-14T08:45:40.464 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2400 IOs, 934104064 bytes 2024-09-14T08:45:40.678 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2500 IOs, 967905280 bytes 2024-09-14T08:45:40.902 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2600 IOs, 1037143040 bytes 2024-09-14T08:45:41.000 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2700 IOs, 1056261120 bytes 2024-09-14T08:45:41.123 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2800 IOs, 1077019648 bytes 2024-09-14T08:45:41.304 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2900 IOs, 1127301120 bytes 2024-09-14T08:45:41.462 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3000 IOs, 1165385728 bytes 2024-09-14T08:45:41.570 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ error_code=124 2024-09-14T08:45:41.571 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set -e 2024-09-14T08:45:41.571 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T08:45:41.571 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:45:41.571 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T08:45:41.571 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_4 2024-09-14T08:45:41.571 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image_and_enable_mirror cluster2 mirror image_4 snapshot 128M 2024-09-14T08:45:41.571 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:45:41.571 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:45:41.571 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:45:41.571 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:45:41.571 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_4 2024-09-14T08:45:41.571 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:45:41.572 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:45:41.572 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n snapshot ']' 2024-09-14T08:45:41.572 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:45:41.572 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image cluster2 mirror image_4 128M 2024-09-14T08:45:41.572 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:45:41.572 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:45:41.572 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:45:41.572 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:45:41.573 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_4 2024-09-14T08:45:41.573 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:45:41.573 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local size=128 2024-09-14T08:45:41.573 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n 128M ']' 2024-09-14T08:45:41.573 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ size=128M 2024-09-14T08:45:41.573 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:45:41.573 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_4 2024-09-14T08:45:41.628 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T08:45:41.628 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:45:41.628 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' image = image ']' 2024-09-14T08:45:41.628 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:45:41.628 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ enable_mirror cluster2 mirror image_4 snapshot 2024-09-14T08:45:41.628 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:45:41.628 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:45:41.628 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_4 2024-09-14T08:45:41.628 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:45:41.628 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_4 snapshot 2024-09-14T08:45:42.531 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Mirroring enabled 2024-09-14T08:45:42.537 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 info mirror/image_4 2024-09-14T08:45:42.577 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_4': 2024-09-14T08:45:42.577 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T08:45:42.577 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:45:42.577 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 1 2024-09-14T08:45:42.577 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 12863f61268a 2024-09-14T08:45:42.577 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.12863f61268a 2024-09-14T08:45:42.577 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:45:42.577 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering 2024-09-14T08:45:42.578 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:45:42.578 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:45:42.578 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:45:41 2024 2024-09-14T08:45:42.578 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:45:41 2024 2024-09-14T08:45:42.578 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:45:41 2024 2024-09-14T08:45:42.578 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:45:42.578 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:45:42.578 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: 3fcd7b58-faf2-4823-8a72-3bd4c17d6db0 2024-09-14T08:45:42.578 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: true 2024-09-14T08:45:42.580 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n '' ']' 2024-09-14T08:45:42.580 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ stress_write_image cluster2 mirror image_4 2024-09-14T08:45:42.580 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:45:42.580 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:45:42.581 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_4 2024-09-14T08:45:42.581 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T08:45:42.582 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local duration=6 2024-09-14T08:45:42.582 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set +e 2024-09-14T08:45:42.582 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ timeout 6s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_4 --debug-rbd=20 --debug-journaler=20 2024-09-14T08:45:42.731 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:100 IOs, 17536000 bytes 2024-09-14T08:45:42.793 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:200 IOs, 19922944 bytes 2024-09-14T08:45:42.939 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:300 IOs, 61516800 bytes 2024-09-14T08:45:43.116 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:400 IOs, 105145344 bytes 2024-09-14T08:45:43.426 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:500 IOs, 208546816 bytes 2024-09-14T08:45:43.589 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:600 IOs, 256462848 bytes 2024-09-14T08:45:43.679 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:700 IOs, 257079296 bytes 2024-09-14T08:45:43.774 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:800 IOs, 272189440 bytes 2024-09-14T08:45:43.963 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:900 IOs, 340262912 bytes 2024-09-14T08:45:44.149 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1000 IOs, 378513408 bytes 2024-09-14T08:45:44.445 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1100 IOs, 488025088 bytes 2024-09-14T08:45:44.622 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1200 IOs, 527666176 bytes 2024-09-14T08:45:44.833 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1300 IOs, 582788096 bytes 2024-09-14T08:45:45.067 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1400 IOs, 676823040 bytes 2024-09-14T08:45:45.248 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1500 IOs, 694916096 bytes 2024-09-14T08:45:45.356 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1600 IOs, 713300992 bytes 2024-09-14T08:45:45.476 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1700 IOs, 737351680 bytes 2024-09-14T08:45:45.624 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1800 IOs, 773235712 bytes 2024-09-14T08:45:45.873 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1900 IOs, 848207872 bytes 2024-09-14T08:45:46.006 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2000 IOs, 883927040 bytes 2024-09-14T08:45:46.184 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2100 IOs, 925524992 bytes 2024-09-14T08:45:46.293 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2200 IOs, 942604288 bytes 2024-09-14T08:45:46.439 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2300 IOs, 981620736 bytes 2024-09-14T08:45:46.556 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2400 IOs, 1027416064 bytes 2024-09-14T08:45:46.718 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2500 IOs, 1042381824 bytes 2024-09-14T08:45:46.821 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2600 IOs, 1071144960 bytes 2024-09-14T08:45:47.072 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2700 IOs, 1133623296 bytes 2024-09-14T08:45:47.205 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2800 IOs, 1174117376 bytes 2024-09-14T08:45:47.382 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2900 IOs, 1208409088 bytes 2024-09-14T08:45:47.559 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3000 IOs, 1248365568 bytes 2024-09-14T08:45:47.693 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3100 IOs, 1277841408 bytes 2024-09-14T08:45:47.832 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3200 IOs, 1310639104 bytes 2024-09-14T08:45:47.900 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3300 IOs, 1311253504 bytes 2024-09-14T08:45:47.993 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3400 IOs, 1324808192 bytes 2024-09-14T08:45:48.123 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3500 IOs, 1352485888 bytes 2024-09-14T08:45:48.259 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3600 IOs, 1380260864 bytes 2024-09-14T08:45:48.487 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3700 IOs, 1455344640 bytes 2024-09-14T08:45:48.588 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ error_code=124 2024-09-14T08:45:48.588 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set -e 2024-09-14T08:45:48.588 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T08:45:48.588 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:45:48.588 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T08:45:48.588 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_5 2024-09-14T08:45:48.588 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image_and_enable_mirror cluster2 mirror image_5 snapshot 128M 2024-09-14T08:45:48.589 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:45:48.589 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:45:48.589 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:45:48.589 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:45:48.589 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_5 2024-09-14T08:45:48.589 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:45:48.589 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:45:48.589 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n snapshot ']' 2024-09-14T08:45:48.589 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:45:48.589 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image cluster2 mirror image_5 128M 2024-09-14T08:45:48.589 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:45:48.589 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:45:48.589 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:45:48.589 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:45:48.589 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_5 2024-09-14T08:45:48.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:45:48.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local size=128 2024-09-14T08:45:48.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n 128M ']' 2024-09-14T08:45:48.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ size=128M 2024-09-14T08:45:48.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:45:48.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_5 2024-09-14T08:45:48.635 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T08:45:48.635 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:45:48.635 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' image = image ']' 2024-09-14T08:45:48.635 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:45:48.635 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ enable_mirror cluster2 mirror image_5 snapshot 2024-09-14T08:45:48.635 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:45:48.635 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:45:48.635 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_5 2024-09-14T08:45:48.635 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:45:48.635 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_5 snapshot 2024-09-14T08:45:49.559 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Mirroring enabled 2024-09-14T08:45:49.565 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 info mirror/image_5 2024-09-14T08:45:49.608 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_5': 2024-09-14T08:45:49.608 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T08:45:49.608 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:45:49.608 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 1 2024-09-14T08:45:49.608 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 1292f4b2a784 2024-09-14T08:45:49.608 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.1292f4b2a784 2024-09-14T08:45:49.608 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:45:49.608 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering 2024-09-14T08:45:49.609 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:45:49.609 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:45:49.609 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:45:48 2024 2024-09-14T08:45:49.609 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:45:48 2024 2024-09-14T08:45:49.609 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:45:48 2024 2024-09-14T08:45:49.609 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:45:49.609 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:45:49.609 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: 98c80276-e6b6-4a70-b145-47505e1b7690 2024-09-14T08:45:49.609 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: true 2024-09-14T08:45:49.611 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n '' ']' 2024-09-14T08:45:49.612 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ stress_write_image cluster2 mirror image_5 2024-09-14T08:45:49.612 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:45:49.612 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:45:49.612 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_5 2024-09-14T08:45:49.612 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T08:45:49.613 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local duration=12 2024-09-14T08:45:49.614 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set +e 2024-09-14T08:45:49.614 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ timeout 12s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_5 --debug-rbd=20 --debug-journaler=20 2024-09-14T08:45:49.832 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:100 IOs, 53256192 bytes 2024-09-14T08:45:49.927 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:200 IOs, 66890752 bytes 2024-09-14T08:45:50.155 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:300 IOs, 137368576 bytes 2024-09-14T08:45:50.337 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:400 IOs, 183607296 bytes 2024-09-14T08:45:50.460 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:500 IOs, 223464448 bytes 2024-09-14T08:45:50.658 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:600 IOs, 265225216 bytes 2024-09-14T08:45:50.867 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:700 IOs, 323475456 bytes 2024-09-14T08:45:51.029 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:800 IOs, 361561088 bytes 2024-09-14T08:45:51.222 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:900 IOs, 416877568 bytes 2024-09-14T08:45:51.330 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1000 IOs, 436045824 bytes 2024-09-14T08:45:51.558 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1100 IOs, 510227456 bytes 2024-09-14T08:45:51.835 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1200 IOs, 592169984 bytes 2024-09-14T08:45:52.067 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1300 IOs, 668280832 bytes 2024-09-14T08:45:52.300 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1400 IOs, 730856448 bytes 2024-09-14T08:45:52.504 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1500 IOs, 790830080 bytes 2024-09-14T08:45:52.656 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1600 IOs, 825811968 bytes 2024-09-14T08:45:52.806 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1700 IOs, 859184128 bytes 2024-09-14T08:45:52.954 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1800 IOs, 891109376 bytes 2024-09-14T08:45:53.285 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1900 IOs, 925036544 bytes 2024-09-14T08:45:53.412 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2000 IOs, 946317312 bytes 2024-09-14T08:45:53.565 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2100 IOs, 984884224 bytes 2024-09-14T08:45:53.849 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2200 IOs, 1073142784 bytes 2024-09-14T08:45:54.004 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2300 IOs, 1105709056 bytes 2024-09-14T08:45:54.140 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2400 IOs, 1130143744 bytes 2024-09-14T08:45:54.318 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2500 IOs, 1178979328 bytes 2024-09-14T08:45:54.431 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2600 IOs, 1209976832 bytes 2024-09-14T08:45:54.566 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2700 IOs, 1224521728 bytes 2024-09-14T08:45:54.733 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2800 IOs, 1264517120 bytes 2024-09-14T08:45:54.908 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2900 IOs, 1308431360 bytes 2024-09-14T08:45:54.973 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3000 IOs, 1318105088 bytes 2024-09-14T08:45:55.108 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3100 IOs, 1352055808 bytes 2024-09-14T08:45:55.246 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3200 IOs, 1373281280 bytes 2024-09-14T08:45:55.505 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3300 IOs, 1445770240 bytes 2024-09-14T08:45:55.688 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3400 IOs, 1510645760 bytes 2024-09-14T08:45:56.039 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3500 IOs, 1605438464 bytes 2024-09-14T08:45:56.187 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3600 IOs, 1651614720 bytes 2024-09-14T08:45:56.344 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3700 IOs, 1678125056 bytes 2024-09-14T08:45:56.683 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3800 IOs, 1718401024 bytes 2024-09-14T08:45:56.870 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3900 IOs, 1763929088 bytes 2024-09-14T08:45:57.072 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4000 IOs, 1820359680 bytes 2024-09-14T08:45:57.185 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4100 IOs, 1841513472 bytes 2024-09-14T08:45:57.389 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4200 IOs, 1893302272 bytes 2024-09-14T08:45:57.540 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4300 IOs, 1926009856 bytes 2024-09-14T08:45:57.657 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4400 IOs, 1935102976 bytes 2024-09-14T08:45:57.829 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4500 IOs, 1983376384 bytes 2024-09-14T08:45:57.903 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4600 IOs, 1983977472 bytes 2024-09-14T08:45:57.906 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.8303035047014644) > weight (0.25) 2024-09-14T08:45:57.906 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.42741720684434237) > weight (0.25) 2024-09-14T08:45:57.906 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.9518665301840601) > weight (0.25) 2024-09-14T08:45:57.906 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.3 with skip (0.4935869312731701) > weight (0.25) 2024-09-14T08:45:57.906 INFO:tasks.rbd_mirror_thrash:waiting for 86 secs before thrashing 2024-09-14T08:45:58.132 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4700 IOs, 2050458624 bytes 2024-09-14T08:45:58.255 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4800 IOs, 2086609920 bytes 2024-09-14T08:45:58.454 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4900 IOs, 2126750720 bytes 2024-09-14T08:45:58.547 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5000 IOs, 2133472256 bytes 2024-09-14T08:45:58.652 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5100 IOs, 2151647232 bytes 2024-09-14T08:45:58.851 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5200 IOs, 2206020608 bytes 2024-09-14T08:45:59.028 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5300 IOs, 2275798016 bytes 2024-09-14T08:45:59.321 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5400 IOs, 2341630976 bytes 2024-09-14T08:45:59.488 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5500 IOs, 2378905600 bytes 2024-09-14T08:45:59.687 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5600 IOs, 2432945152 bytes 2024-09-14T08:45:59.781 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5700 IOs, 2433554432 bytes 2024-09-14T08:46:00.064 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5800 IOs, 2527162368 bytes 2024-09-14T08:46:00.202 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5900 IOs, 2560207872 bytes 2024-09-14T08:46:00.364 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6000 IOs, 2582833152 bytes 2024-09-14T08:46:00.557 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6100 IOs, 2634413056 bytes 2024-09-14T08:46:00.693 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6200 IOs, 2662695936 bytes 2024-09-14T08:46:00.994 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6300 IOs, 2779033600 bytes 2024-09-14T08:46:01.202 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6400 IOs, 2817576960 bytes 2024-09-14T08:46:01.437 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6500 IOs, 2891860992 bytes 2024-09-14T08:46:01.536 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6600 IOs, 2895926272 bytes 2024-09-14T08:46:01.619 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ error_code=124 2024-09-14T08:46:01.619 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set -e 2024-09-14T08:46:01.620 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T08:46:01.620 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:46:01.620 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T08:46:01.620 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_6 2024-09-14T08:46:01.620 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image_and_enable_mirror cluster2 mirror image_6 snapshot 128M 2024-09-14T08:46:01.620 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:46:01.620 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:46:01.620 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:46:01.620 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:46:01.620 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_6 2024-09-14T08:46:01.620 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:46:01.620 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:46:01.620 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n snapshot ']' 2024-09-14T08:46:01.621 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:46:01.621 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image cluster2 mirror image_6 128M 2024-09-14T08:46:01.621 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:46:01.621 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:46:01.621 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:46:01.621 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:46:01.621 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_6 2024-09-14T08:46:01.621 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:46:01.621 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local size=128 2024-09-14T08:46:01.621 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n 128M ']' 2024-09-14T08:46:01.621 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ size=128M 2024-09-14T08:46:01.621 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:46:01.621 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_6 2024-09-14T08:46:01.674 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T08:46:01.674 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:46:01.674 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' image = image ']' 2024-09-14T08:46:01.675 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:46:01.675 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ enable_mirror cluster2 mirror image_6 snapshot 2024-09-14T08:46:01.675 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:46:01.675 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:46:01.675 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_6 2024-09-14T08:46:01.675 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:46:01.675 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_6 snapshot 2024-09-14T08:46:02.070 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Mirroring enabled 2024-09-14T08:46:02.076 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 info mirror/image_6 2024-09-14T08:46:02.118 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_6': 2024-09-14T08:46:02.119 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T08:46:02.119 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:46:02.119 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 1 2024-09-14T08:46:02.119 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 129ed32d21d 2024-09-14T08:46:02.119 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.129ed32d21d 2024-09-14T08:46:02.119 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:46:02.119 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering 2024-09-14T08:46:02.119 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:46:02.119 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:46:02.119 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:46:01 2024 2024-09-14T08:46:02.119 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:46:01 2024 2024-09-14T08:46:02.119 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:46:01 2024 2024-09-14T08:46:02.119 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:46:02.120 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:46:02.120 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: 82e82fc9-f6e1-4a89-9145-f0e06f4942c1 2024-09-14T08:46:02.120 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: true 2024-09-14T08:46:02.122 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n '' ']' 2024-09-14T08:46:02.122 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ stress_write_image cluster2 mirror image_6 2024-09-14T08:46:02.122 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:46:02.122 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:46:02.123 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_6 2024-09-14T08:46:02.123 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T08:46:02.124 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local duration=5 2024-09-14T08:46:02.124 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set +e 2024-09-14T08:46:02.124 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ timeout 5s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_6 --debug-rbd=20 --debug-journaler=20 2024-09-14T08:46:02.348 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:100 IOs, 49006592 bytes 2024-09-14T08:46:02.702 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:200 IOs, 169498624 bytes 2024-09-14T08:46:02.816 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:300 IOs, 191117312 bytes 2024-09-14T08:46:03.062 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:400 IOs, 264617984 bytes 2024-09-14T08:46:03.251 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:500 IOs, 318388224 bytes 2024-09-14T08:46:03.347 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:600 IOs, 337521664 bytes 2024-09-14T08:46:03.604 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:700 IOs, 412493824 bytes 2024-09-14T08:46:03.734 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:800 IOs, 441185280 bytes 2024-09-14T08:46:03.881 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:900 IOs, 482948096 bytes 2024-09-14T08:46:04.039 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1000 IOs, 509499392 bytes 2024-09-14T08:46:04.168 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1100 IOs, 538094592 bytes 2024-09-14T08:46:04.390 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1200 IOs, 625918976 bytes 2024-09-14T08:46:04.629 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1300 IOs, 677433344 bytes 2024-09-14T08:46:04.747 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1400 IOs, 695644160 bytes 2024-09-14T08:46:04.851 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1500 IOs, 715649024 bytes 2024-09-14T08:46:05.060 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1600 IOs, 791766016 bytes 2024-09-14T08:46:05.176 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1700 IOs, 796982272 bytes 2024-09-14T08:46:05.340 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1800 IOs, 862819328 bytes 2024-09-14T08:46:05.573 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1900 IOs, 929699840 bytes 2024-09-14T08:46:05.815 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2000 IOs, 987353088 bytes 2024-09-14T08:46:05.933 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2100 IOs, 1009187840 bytes 2024-09-14T08:46:06.050 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2200 IOs, 1027219456 bytes 2024-09-14T08:46:06.116 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2300 IOs, 1027858432 bytes 2024-09-14T08:46:06.236 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2400 IOs, 1056947200 bytes 2024-09-14T08:46:06.396 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2500 IOs, 1108453376 bytes 2024-09-14T08:46:06.550 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2600 IOs, 1125572608 bytes 2024-09-14T08:46:06.695 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2700 IOs, 1158226944 bytes 2024-09-14T08:46:06.763 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2800 IOs, 1158840320 bytes 2024-09-14T08:46:06.830 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2900 IOs, 1159449600 bytes 2024-09-14T08:46:06.950 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3000 IOs, 1180973056 bytes 2024-09-14T08:46:07.115 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3100 IOs, 1223267328 bytes 2024-09-14T08:46:07.132 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ error_code=124 2024-09-14T08:46:07.132 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set -e 2024-09-14T08:46:07.132 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T08:46:07.132 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:46:07.132 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T08:46:07.132 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_7 2024-09-14T08:46:07.132 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image_and_enable_mirror cluster2 mirror image_7 snapshot 128M 2024-09-14T08:46:07.132 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:46:07.133 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:46:07.133 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:46:07.133 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:46:07.133 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_7 2024-09-14T08:46:07.133 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:46:07.133 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:46:07.133 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n snapshot ']' 2024-09-14T08:46:07.133 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:46:07.133 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image cluster2 mirror image_7 128M 2024-09-14T08:46:07.133 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:46:07.133 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:46:07.133 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:46:07.134 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:46:07.134 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_7 2024-09-14T08:46:07.134 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:46:07.134 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local size=128 2024-09-14T08:46:07.134 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n 128M ']' 2024-09-14T08:46:07.134 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ size=128M 2024-09-14T08:46:07.134 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:46:07.134 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_7 2024-09-14T08:46:07.183 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T08:46:07.183 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:46:07.183 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' image = image ']' 2024-09-14T08:46:07.183 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:46:07.183 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ enable_mirror cluster2 mirror image_7 snapshot 2024-09-14T08:46:07.184 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:46:07.184 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:46:07.184 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_7 2024-09-14T08:46:07.184 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:46:07.184 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_7 snapshot 2024-09-14T08:46:08.099 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Mirroring enabled 2024-09-14T08:46:08.105 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 info mirror/image_7 2024-09-14T08:46:08.348 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_7': 2024-09-14T08:46:08.348 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T08:46:08.348 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:46:08.349 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 1 2024-09-14T08:46:08.349 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 12aa3ae78fbd 2024-09-14T08:46:08.349 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.12aa3ae78fbd 2024-09-14T08:46:08.349 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:46:08.349 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering 2024-09-14T08:46:08.349 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:46:08.349 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:46:08.349 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:46:07 2024 2024-09-14T08:46:08.349 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:46:07 2024 2024-09-14T08:46:08.349 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:46:07 2024 2024-09-14T08:46:08.349 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:46:08.349 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:46:08.349 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: 46c72cd0-0717-48e2-bd75-e9b03e063bb7 2024-09-14T08:46:08.349 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: true 2024-09-14T08:46:08.352 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n '' ']' 2024-09-14T08:46:08.352 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ stress_write_image cluster2 mirror image_7 2024-09-14T08:46:08.352 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:46:08.352 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:46:08.352 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_7 2024-09-14T08:46:08.352 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T08:46:08.354 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local duration=6 2024-09-14T08:46:08.354 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set +e 2024-09-14T08:46:08.354 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ timeout 6s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_7 --debug-rbd=20 --debug-journaler=20 2024-09-14T08:46:08.478 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:100 IOs, 605184 bytes 2024-09-14T08:46:08.587 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:200 IOs, 32242688 bytes 2024-09-14T08:46:08.731 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:300 IOs, 68350976 bytes 2024-09-14T08:46:08.878 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:400 IOs, 89526272 bytes 2024-09-14T08:46:08.961 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:500 IOs, 97444864 bytes 2024-09-14T08:46:09.127 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:600 IOs, 138040320 bytes 2024-09-14T08:46:09.274 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:700 IOs, 169153536 bytes 2024-09-14T08:46:09.387 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:800 IOs, 190734336 bytes 2024-09-14T08:46:09.475 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:900 IOs, 224521216 bytes 2024-09-14T08:46:09.654 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1000 IOs, 246834176 bytes 2024-09-14T08:46:09.838 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1100 IOs, 299833344 bytes 2024-09-14T08:46:09.949 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1200 IOs, 323457024 bytes 2024-09-14T08:46:10.174 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1300 IOs, 380256256 bytes 2024-09-14T08:46:10.333 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1400 IOs, 417372160 bytes 2024-09-14T08:46:10.488 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1500 IOs, 441975808 bytes 2024-09-14T08:46:10.647 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1600 IOs, 480332800 bytes 2024-09-14T08:46:10.865 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1700 IOs, 541784064 bytes 2024-09-14T08:46:10.965 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1800 IOs, 564712448 bytes 2024-09-14T08:46:11.202 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1900 IOs, 626621440 bytes 2024-09-14T08:46:11.328 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2000 IOs, 650143744 bytes 2024-09-14T08:46:11.468 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2100 IOs, 683017216 bytes 2024-09-14T08:46:11.652 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2200 IOs, 745643008 bytes 2024-09-14T08:46:11.810 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2300 IOs, 770742272 bytes 2024-09-14T08:46:12.025 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2400 IOs, 827234304 bytes 2024-09-14T08:46:12.344 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2500 IOs, 930905088 bytes 2024-09-14T08:46:12.616 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2600 IOs, 1023985664 bytes 2024-09-14T08:46:12.826 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2700 IOs, 1078350848 bytes 2024-09-14T08:46:12.910 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2800 IOs, 1084070912 bytes 2024-09-14T08:46:13.100 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2900 IOs, 1144702976 bytes 2024-09-14T08:46:13.238 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3000 IOs, 1165906944 bytes 2024-09-14T08:46:13.401 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3100 IOs, 1198799872 bytes 2024-09-14T08:46:13.554 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3200 IOs, 1232616448 bytes 2024-09-14T08:46:13.719 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3300 IOs, 1271865344 bytes 2024-09-14T08:46:13.852 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3400 IOs, 1319021568 bytes 2024-09-14T08:46:14.125 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3500 IOs, 1384936448 bytes 2024-09-14T08:46:14.284 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3600 IOs, 1415738368 bytes 2024-09-14T08:46:14.360 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ error_code=124 2024-09-14T08:46:14.360 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set -e 2024-09-14T08:46:14.360 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T08:46:14.361 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:46:14.361 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T08:46:14.361 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_8 2024-09-14T08:46:14.361 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image_and_enable_mirror cluster2 mirror image_8 snapshot 128M 2024-09-14T08:46:14.361 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:46:14.361 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:46:14.361 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:46:14.361 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:46:14.361 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_8 2024-09-14T08:46:14.361 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:46:14.361 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:46:14.361 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n snapshot ']' 2024-09-14T08:46:14.361 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:46:14.361 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image cluster2 mirror image_8 128M 2024-09-14T08:46:14.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:46:14.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:46:14.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:46:14.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:46:14.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_8 2024-09-14T08:46:14.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:46:14.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local size=128 2024-09-14T08:46:14.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n 128M ']' 2024-09-14T08:46:14.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ size=128M 2024-09-14T08:46:14.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:46:14.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_8 2024-09-14T08:46:14.428 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T08:46:14.428 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:46:14.428 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' image = image ']' 2024-09-14T08:46:14.428 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:46:14.428 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ enable_mirror cluster2 mirror image_8 snapshot 2024-09-14T08:46:14.429 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:46:14.429 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:46:14.429 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_8 2024-09-14T08:46:14.429 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:46:14.429 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_8 snapshot 2024-09-14T08:46:15.114 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Mirroring enabled 2024-09-14T08:46:15.119 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 info mirror/image_8 2024-09-14T08:46:15.161 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_8': 2024-09-14T08:46:15.161 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T08:46:15.161 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:46:15.161 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 1 2024-09-14T08:46:15.161 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 12b613ed156a 2024-09-14T08:46:15.161 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.12b613ed156a 2024-09-14T08:46:15.161 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:46:15.161 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering 2024-09-14T08:46:15.162 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:46:15.162 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:46:15.162 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:46:14 2024 2024-09-14T08:46:15.162 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:46:14 2024 2024-09-14T08:46:15.162 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:46:14 2024 2024-09-14T08:46:15.162 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:46:15.162 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:46:15.162 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: 9b43aad5-e6f1-4a31-a25c-d7df22bd01da 2024-09-14T08:46:15.162 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: true 2024-09-14T08:46:15.165 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n '' ']' 2024-09-14T08:46:15.165 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ stress_write_image cluster2 mirror image_8 2024-09-14T08:46:15.165 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:46:15.165 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:46:15.165 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_8 2024-09-14T08:46:15.165 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T08:46:15.166 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local duration=6 2024-09-14T08:46:15.167 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set +e 2024-09-14T08:46:15.167 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ timeout 6s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_8 --debug-rbd=20 --debug-journaler=20 2024-09-14T08:46:15.396 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:100 IOs, 53908480 bytes 2024-09-14T08:46:15.550 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:200 IOs, 95098880 bytes 2024-09-14T08:46:15.681 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:300 IOs, 109907968 bytes 2024-09-14T08:46:15.755 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:400 IOs, 119493632 bytes 2024-09-14T08:46:15.929 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:500 IOs, 164630528 bytes 2024-09-14T08:46:16.111 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:600 IOs, 219341824 bytes 2024-09-14T08:46:16.242 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:700 IOs, 235455488 bytes 2024-09-14T08:46:16.371 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:800 IOs, 261882880 bytes 2024-09-14T08:46:16.560 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:900 IOs, 314226688 bytes 2024-09-14T08:46:16.676 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1000 IOs, 335477760 bytes 2024-09-14T08:46:16.942 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1100 IOs, 413024256 bytes 2024-09-14T08:46:17.062 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1200 IOs, 435516416 bytes 2024-09-14T08:46:17.214 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1300 IOs, 462751744 bytes 2024-09-14T08:46:17.418 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1400 IOs, 523005952 bytes 2024-09-14T08:46:17.679 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1500 IOs, 603375616 bytes 2024-09-14T08:46:17.887 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1600 IOs, 648153088 bytes 2024-09-14T08:46:18.052 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1700 IOs, 698838016 bytes 2024-09-14T08:46:18.278 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1800 IOs, 750531584 bytes 2024-09-14T08:46:18.527 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1900 IOs, 848032768 bytes 2024-09-14T08:46:18.735 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2000 IOs, 897047552 bytes 2024-09-14T08:46:18.945 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2100 IOs, 951488512 bytes 2024-09-14T08:46:19.154 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2200 IOs, 1008301056 bytes 2024-09-14T08:46:19.326 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2300 IOs, 1052926976 bytes 2024-09-14T08:46:19.544 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2400 IOs, 1114370048 bytes 2024-09-14T08:46:19.688 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2500 IOs, 1137339392 bytes 2024-09-14T08:46:19.913 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2600 IOs, 1196058624 bytes 2024-09-14T08:46:19.987 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2700 IOs, 1199556608 bytes 2024-09-14T08:46:20.078 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2800 IOs, 1208532992 bytes 2024-09-14T08:46:20.237 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2900 IOs, 1245095936 bytes 2024-09-14T08:46:20.426 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3000 IOs, 1300417536 bytes 2024-09-14T08:46:20.557 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3100 IOs, 1322151936 bytes 2024-09-14T08:46:20.656 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3200 IOs, 1334193152 bytes 2024-09-14T08:46:20.768 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3300 IOs, 1352216576 bytes 2024-09-14T08:46:20.989 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3400 IOs, 1418593280 bytes 2024-09-14T08:46:21.164 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3500 IOs, 1460081664 bytes 2024-09-14T08:46:21.172 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ error_code=124 2024-09-14T08:46:21.172 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set -e 2024-09-14T08:46:21.172 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T08:46:21.172 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:46:21.172 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T08:46:21.173 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_9 2024-09-14T08:46:21.173 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image_and_enable_mirror cluster2 mirror image_9 snapshot 128M 2024-09-14T08:46:21.173 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:46:21.173 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:46:21.173 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:46:21.173 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:46:21.173 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_9 2024-09-14T08:46:21.173 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:46:21.173 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:46:21.173 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n snapshot ']' 2024-09-14T08:46:21.173 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:46:21.174 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image cluster2 mirror image_9 128M 2024-09-14T08:46:21.174 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:46:21.174 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:46:21.174 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:46:21.174 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:46:21.174 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_9 2024-09-14T08:46:21.174 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:46:21.174 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local size=128 2024-09-14T08:46:21.174 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n 128M ']' 2024-09-14T08:46:21.174 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ size=128M 2024-09-14T08:46:21.174 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:46:21.174 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_9 2024-09-14T08:46:21.219 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T08:46:21.219 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:46:21.219 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' image = image ']' 2024-09-14T08:46:21.219 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:46:21.219 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ enable_mirror cluster2 mirror image_9 snapshot 2024-09-14T08:46:21.219 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:46:21.219 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:46:21.219 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_9 2024-09-14T08:46:21.219 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:46:21.219 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_9 snapshot 2024-09-14T08:46:22.129 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Mirroring enabled 2024-09-14T08:46:22.134 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 info mirror/image_9 2024-09-14T08:46:22.175 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_9': 2024-09-14T08:46:22.175 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T08:46:22.175 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:46:22.176 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 1 2024-09-14T08:46:22.176 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 12c2f3c1a65f 2024-09-14T08:46:22.176 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.12c2f3c1a65f 2024-09-14T08:46:22.176 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:46:22.176 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering 2024-09-14T08:46:22.176 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:46:22.176 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:46:22.176 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:46:21 2024 2024-09-14T08:46:22.176 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:46:21 2024 2024-09-14T08:46:22.176 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:46:21 2024 2024-09-14T08:46:22.176 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:46:22.177 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:46:22.177 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: 6b16cbe5-ac26-4754-a383-29c75eb5ef3c 2024-09-14T08:46:22.177 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: true 2024-09-14T08:46:22.179 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n '' ']' 2024-09-14T08:46:22.179 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ stress_write_image cluster2 mirror image_9 2024-09-14T08:46:22.180 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:46:22.180 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:46:22.180 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_9 2024-09-14T08:46:22.180 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T08:46:22.181 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local duration=6 2024-09-14T08:46:22.181 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set +e 2024-09-14T08:46:22.181 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ timeout 6s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_9 --debug-rbd=20 --debug-journaler=20 2024-09-14T08:46:22.389 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:100 IOs, 40118272 bytes 2024-09-14T08:46:22.569 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:200 IOs, 87477248 bytes 2024-09-14T08:46:22.665 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:300 IOs, 126209024 bytes 2024-09-14T08:46:22.946 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:400 IOs, 185148416 bytes 2024-09-14T08:46:23.104 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:500 IOs, 228207616 bytes 2024-09-14T08:46:23.315 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:600 IOs, 280437760 bytes 2024-09-14T08:46:23.470 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:700 IOs, 328763392 bytes 2024-09-14T08:46:23.657 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:800 IOs, 367286272 bytes 2024-09-14T08:46:23.752 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:900 IOs, 377896960 bytes 2024-09-14T08:46:23.864 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1000 IOs, 417202176 bytes 2024-09-14T08:46:24.082 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1100 IOs, 454767616 bytes 2024-09-14T08:46:24.214 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1200 IOs, 479745024 bytes 2024-09-14T08:46:24.331 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1300 IOs, 500154368 bytes 2024-09-14T08:46:24.446 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1400 IOs, 518034432 bytes 2024-09-14T08:46:24.634 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1500 IOs, 578603008 bytes 2024-09-14T08:46:24.774 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1600 IOs, 598754304 bytes 2024-09-14T08:46:25.070 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1700 IOs, 705499136 bytes 2024-09-14T08:46:25.220 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1800 IOs, 731442176 bytes 2024-09-14T08:46:25.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1900 IOs, 762544128 bytes 2024-09-14T08:46:25.433 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2000 IOs, 769853440 bytes 2024-09-14T08:46:25.693 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2100 IOs, 844998656 bytes 2024-09-14T08:46:25.892 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2200 IOs, 899846144 bytes 2024-09-14T08:46:26.189 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2300 IOs, 1003314176 bytes 2024-09-14T08:46:26.349 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2400 IOs, 1035115520 bytes 2024-09-14T08:46:26.479 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2500 IOs, 1079513088 bytes 2024-09-14T08:46:26.686 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2600 IOs, 1126315008 bytes 2024-09-14T08:46:26.767 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2700 IOs, 1126938624 bytes 2024-09-14T08:46:26.940 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2800 IOs, 1187258368 bytes 2024-09-14T08:46:27.092 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2900 IOs, 1213129728 bytes 2024-09-14T08:46:27.287 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3000 IOs, 1260879872 bytes 2024-09-14T08:46:27.474 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3100 IOs, 1310745600 bytes 2024-09-14T08:46:27.719 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3200 IOs, 1387092992 bytes 2024-09-14T08:46:27.905 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3300 IOs, 1433437184 bytes 2024-09-14T08:46:28.097 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3400 IOs, 1486950400 bytes 2024-09-14T08:46:28.189 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ error_code=124 2024-09-14T08:46:28.189 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set -e 2024-09-14T08:46:28.189 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T08:46:28.189 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:46:28.190 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T08:46:28.190 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_10 2024-09-14T08:46:28.190 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image_and_enable_mirror cluster2 mirror image_10 snapshot 128M 2024-09-14T08:46:28.190 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:46:28.190 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:46:28.190 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:46:28.190 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:46:28.190 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_10 2024-09-14T08:46:28.190 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:46:28.190 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:46:28.190 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n snapshot ']' 2024-09-14T08:46:28.190 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:46:28.191 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image cluster2 mirror image_10 128M 2024-09-14T08:46:28.191 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:46:28.191 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:46:28.191 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:46:28.191 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:46:28.191 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_10 2024-09-14T08:46:28.191 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:46:28.191 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local size=128 2024-09-14T08:46:28.191 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n 128M ']' 2024-09-14T08:46:28.191 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ size=128M 2024-09-14T08:46:28.191 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:46:28.191 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_10 2024-09-14T08:46:28.240 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T08:46:28.241 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:46:28.241 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' image = image ']' 2024-09-14T08:46:28.241 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:46:28.241 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ enable_mirror cluster2 mirror image_10 snapshot 2024-09-14T08:46:28.241 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:46:28.241 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:46:28.241 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_10 2024-09-14T08:46:28.241 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:46:28.241 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_10 snapshot 2024-09-14T08:46:29.147 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Mirroring enabled 2024-09-14T08:46:29.153 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 info mirror/image_10 2024-09-14T08:46:29.196 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_10': 2024-09-14T08:46:29.196 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T08:46:29.196 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:46:29.196 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 1 2024-09-14T08:46:29.196 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 12ce37e438f3 2024-09-14T08:46:29.197 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.12ce37e438f3 2024-09-14T08:46:29.197 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:46:29.197 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering 2024-09-14T08:46:29.197 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:46:29.197 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:46:29.197 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:46:28 2024 2024-09-14T08:46:29.197 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:46:28 2024 2024-09-14T08:46:29.197 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:46:28 2024 2024-09-14T08:46:29.197 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:46:29.197 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:46:29.197 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: 0a80ae74-9f85-4745-bf89-a713c5afc298 2024-09-14T08:46:29.197 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: true 2024-09-14T08:46:29.200 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n '' ']' 2024-09-14T08:46:29.200 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ stress_write_image cluster2 mirror image_10 2024-09-14T08:46:29.200 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:46:29.200 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:46:29.200 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_10 2024-09-14T08:46:29.200 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T08:46:29.202 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local duration=14 2024-09-14T08:46:29.202 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set +e 2024-09-14T08:46:29.203 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ timeout 14s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_10 --debug-rbd=20 --debug-journaler=20 2024-09-14T08:46:29.500 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:100 IOs, 84644864 bytes 2024-09-14T08:46:29.675 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:200 IOs, 134953984 bytes 2024-09-14T08:46:29.841 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:300 IOs, 184518656 bytes 2024-09-14T08:46:29.965 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:400 IOs, 191892480 bytes 2024-09-14T08:46:30.058 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:500 IOs, 216593408 bytes 2024-09-14T08:46:30.238 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:600 IOs, 254330880 bytes 2024-09-14T08:46:30.326 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:700 IOs, 257915904 bytes 2024-09-14T08:46:30.458 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:800 IOs, 287243264 bytes 2024-09-14T08:46:30.623 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:900 IOs, 329677824 bytes 2024-09-14T08:46:30.776 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1000 IOs, 371679232 bytes 2024-09-14T08:46:30.875 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1100 IOs, 377579520 bytes 2024-09-14T08:46:31.038 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1200 IOs, 428932096 bytes 2024-09-14T08:46:31.186 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1300 IOs, 451068928 bytes 2024-09-14T08:46:31.294 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1400 IOs, 469628928 bytes 2024-09-14T08:46:31.581 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1500 IOs, 553195520 bytes 2024-09-14T08:46:31.705 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1600 IOs, 587474944 bytes 2024-09-14T08:46:31.900 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1700 IOs, 632493056 bytes 2024-09-14T08:46:31.993 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1800 IOs, 655229952 bytes 2024-09-14T08:46:32.174 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1900 IOs, 695311360 bytes 2024-09-14T08:46:32.292 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2000 IOs, 714895360 bytes 2024-09-14T08:46:32.430 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2100 IOs, 743507968 bytes 2024-09-14T08:46:32.541 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2200 IOs, 759271424 bytes 2024-09-14T08:46:32.711 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2300 IOs, 781119488 bytes 2024-09-14T08:46:32.889 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2400 IOs, 808744960 bytes 2024-09-14T08:46:33.144 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2500 IOs, 890182656 bytes 2024-09-14T08:46:33.241 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2600 IOs, 897559552 bytes 2024-09-14T08:46:33.313 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2700 IOs, 899441664 bytes 2024-09-14T08:46:33.466 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2800 IOs, 938921984 bytes 2024-09-14T08:46:33.535 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2900 IOs, 939560960 bytes 2024-09-14T08:46:33.669 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3000 IOs, 966670336 bytes 2024-09-14T08:46:33.882 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3100 IOs, 1026013184 bytes 2024-09-14T08:46:34.036 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3200 IOs, 1048083456 bytes 2024-09-14T08:46:34.139 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3300 IOs, 1063252992 bytes 2024-09-14T08:46:34.390 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3400 IOs, 1141360640 bytes 2024-09-14T08:46:34.748 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3500 IOs, 1242346496 bytes 2024-09-14T08:46:35.044 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3600 IOs, 1351179264 bytes 2024-09-14T08:46:35.232 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3700 IOs, 1391238144 bytes 2024-09-14T08:46:35.528 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3800 IOs, 1516324864 bytes 2024-09-14T08:46:35.865 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3900 IOs, 1606482944 bytes 2024-09-14T08:46:36.091 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4000 IOs, 1671528448 bytes 2024-09-14T08:46:36.152 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4100 IOs, 1672138752 bytes 2024-09-14T08:46:36.297 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4200 IOs, 1706158080 bytes 2024-09-14T08:46:36.445 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4300 IOs, 1742070784 bytes 2024-09-14T08:46:36.643 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4400 IOs, 1797305344 bytes 2024-09-14T08:46:36.739 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4500 IOs, 1810026496 bytes 2024-09-14T08:46:36.955 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4600 IOs, 1885276160 bytes 2024-09-14T08:46:37.098 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4700 IOs, 1910776832 bytes 2024-09-14T08:46:37.210 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4800 IOs, 1932422144 bytes 2024-09-14T08:46:37.310 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4900 IOs, 1949769728 bytes 2024-09-14T08:46:37.523 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5000 IOs, 2010103808 bytes 2024-09-14T08:46:37.703 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5100 IOs, 2056177664 bytes 2024-09-14T08:46:37.888 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5200 IOs, 2111699968 bytes 2024-09-14T08:46:38.066 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5300 IOs, 2161087488 bytes 2024-09-14T08:46:38.411 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5400 IOs, 2260044800 bytes 2024-09-14T08:46:38.550 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5500 IOs, 2294982656 bytes 2024-09-14T08:46:38.666 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5600 IOs, 2319000576 bytes 2024-09-14T08:46:38.829 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5700 IOs, 2358845440 bytes 2024-09-14T08:46:38.963 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5800 IOs, 2388828160 bytes 2024-09-14T08:46:39.072 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5900 IOs, 2405406720 bytes 2024-09-14T08:46:39.212 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6000 IOs, 2435748864 bytes 2024-09-14T08:46:39.398 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6100 IOs, 2485397504 bytes 2024-09-14T08:46:39.559 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6200 IOs, 2537441280 bytes 2024-09-14T08:46:39.755 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6300 IOs, 2582861824 bytes 2024-09-14T08:46:39.953 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6400 IOs, 2636369920 bytes 2024-09-14T08:46:40.136 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6500 IOs, 2693608448 bytes 2024-09-14T08:46:40.256 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6600 IOs, 2712211456 bytes 2024-09-14T08:46:40.370 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6700 IOs, 2730360832 bytes 2024-09-14T08:46:40.536 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6800 IOs, 2816296960 bytes 2024-09-14T08:46:40.774 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6900 IOs, 2838417408 bytes 2024-09-14T08:46:41.052 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:7000 IOs, 2928498688 bytes 2024-09-14T08:46:41.328 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:7100 IOs, 3007289344 bytes 2024-09-14T08:46:41.457 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:7200 IOs, 3030830080 bytes 2024-09-14T08:46:41.575 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:7300 IOs, 3054604288 bytes 2024-09-14T08:46:41.670 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:7400 IOs, 3068558336 bytes 2024-09-14T08:46:41.807 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:7500 IOs, 3106739200 bytes 2024-09-14T08:46:41.988 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:7600 IOs, 3148033024 bytes 2024-09-14T08:46:42.119 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:7700 IOs, 3171533824 bytes 2024-09-14T08:46:42.281 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:7800 IOs, 3227941888 bytes 2024-09-14T08:46:42.581 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:7900 IOs, 3316869120 bytes 2024-09-14T08:46:42.810 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:8000 IOs, 3393312768 bytes 2024-09-14T08:46:42.899 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:8100 IOs, 3396045824 bytes 2024-09-14T08:46:42.967 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:8200 IOs, 3399441408 bytes 2024-09-14T08:46:43.096 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:8300 IOs, 3440196608 bytes 2024-09-14T08:46:43.209 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ error_code=124 2024-09-14T08:46:43.209 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set -e 2024-09-14T08:46:43.209 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T08:46:43.209 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:46:43.209 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T08:46:43.209 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_11 2024-09-14T08:46:43.209 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image_and_enable_mirror cluster2 mirror image_11 snapshot 128M 2024-09-14T08:46:43.209 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:46:43.210 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:46:43.210 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:46:43.210 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:46:43.210 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_11 2024-09-14T08:46:43.210 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:46:43.210 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:46:43.210 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n snapshot ']' 2024-09-14T08:46:43.210 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:46:43.210 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image cluster2 mirror image_11 128M 2024-09-14T08:46:43.210 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:46:43.210 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:46:43.210 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:46:43.210 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:46:43.210 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_11 2024-09-14T08:46:43.210 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:46:43.211 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local size=128 2024-09-14T08:46:43.211 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n 128M ']' 2024-09-14T08:46:43.211 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ size=128M 2024-09-14T08:46:43.211 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:46:43.211 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_11 2024-09-14T08:46:43.256 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T08:46:43.256 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:46:43.256 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' image = image ']' 2024-09-14T08:46:43.256 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:46:43.256 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ enable_mirror cluster2 mirror image_11 snapshot 2024-09-14T08:46:43.257 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:46:43.257 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:46:43.257 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_11 2024-09-14T08:46:43.257 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:46:43.257 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_11 snapshot 2024-09-14T08:46:44.191 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Mirroring enabled 2024-09-14T08:46:44.196 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 info mirror/image_11 2024-09-14T08:46:44.236 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_11': 2024-09-14T08:46:44.236 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T08:46:44.236 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:46:44.237 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 1 2024-09-14T08:46:44.237 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 12da22e8d0ea 2024-09-14T08:46:44.237 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.12da22e8d0ea 2024-09-14T08:46:44.237 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:46:44.237 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering 2024-09-14T08:46:44.237 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:46:44.237 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:46:44.237 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:46:43 2024 2024-09-14T08:46:44.237 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:46:43 2024 2024-09-14T08:46:44.237 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:46:43 2024 2024-09-14T08:46:44.237 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:46:44.237 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:46:44.238 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: f2624247-ccb3-494f-a736-10ddf00ce6ae 2024-09-14T08:46:44.238 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: true 2024-09-14T08:46:44.240 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n '' ']' 2024-09-14T08:46:44.240 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ stress_write_image cluster2 mirror image_11 2024-09-14T08:46:44.240 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:46:44.240 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:46:44.241 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_11 2024-09-14T08:46:44.241 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T08:46:44.242 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local duration=8 2024-09-14T08:46:44.242 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set +e 2024-09-14T08:46:44.242 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ timeout 8s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_11 --debug-rbd=20 --debug-journaler=20 2024-09-14T08:46:44.364 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:100 IOs, 29957120 bytes 2024-09-14T08:46:44.531 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:200 IOs, 53991424 bytes 2024-09-14T08:46:44.624 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:300 IOs, 69724160 bytes 2024-09-14T08:46:44.894 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:400 IOs, 162246656 bytes 2024-09-14T08:46:45.105 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:500 IOs, 228517888 bytes 2024-09-14T08:46:45.244 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:600 IOs, 257923072 bytes 2024-09-14T08:46:45.401 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:700 IOs, 299218944 bytes 2024-09-14T08:46:45.514 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:800 IOs, 329450496 bytes 2024-09-14T08:46:45.697 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:900 IOs, 381619200 bytes 2024-09-14T08:46:45.926 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1000 IOs, 446007296 bytes 2024-09-14T08:46:45.999 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1100 IOs, 457162752 bytes 2024-09-14T08:46:46.344 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1200 IOs, 569640960 bytes 2024-09-14T08:46:46.548 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1300 IOs, 644278272 bytes 2024-09-14T08:46:46.696 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1400 IOs, 666143744 bytes 2024-09-14T08:46:46.857 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1500 IOs, 707123200 bytes 2024-09-14T08:46:46.989 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1600 IOs, 735709184 bytes 2024-09-14T08:46:47.081 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1700 IOs, 749646848 bytes 2024-09-14T08:46:47.286 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1800 IOs, 805637120 bytes 2024-09-14T08:46:47.384 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1900 IOs, 824159232 bytes 2024-09-14T08:46:47.446 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2000 IOs, 824748032 bytes 2024-09-14T08:46:47.639 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2100 IOs, 880784384 bytes 2024-09-14T08:46:47.742 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2200 IOs, 905303040 bytes 2024-09-14T08:46:47.911 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2300 IOs, 946618368 bytes 2024-09-14T08:46:48.080 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2400 IOs, 988073984 bytes 2024-09-14T08:46:48.225 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2500 IOs, 1017102336 bytes 2024-09-14T08:46:48.344 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2600 IOs, 1042599936 bytes 2024-09-14T08:46:48.502 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2700 IOs, 1083956224 bytes 2024-09-14T08:46:48.635 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2800 IOs, 1113092096 bytes 2024-09-14T08:46:48.736 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2900 IOs, 1128638464 bytes 2024-09-14T08:46:48.921 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3000 IOs, 1178187776 bytes 2024-09-14T08:46:49.235 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3100 IOs, 1288164352 bytes 2024-09-14T08:46:49.391 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3200 IOs, 1327432704 bytes 2024-09-14T08:46:49.487 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3300 IOs, 1343946752 bytes 2024-09-14T08:46:49.552 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3400 IOs, 1344559104 bytes 2024-09-14T08:46:49.700 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3500 IOs, 1403430912 bytes 2024-09-14T08:46:49.884 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3600 IOs, 1441255424 bytes 2024-09-14T08:46:50.187 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3700 IOs, 1538949120 bytes 2024-09-14T08:46:50.416 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3800 IOs, 1608816640 bytes 2024-09-14T08:46:50.545 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3900 IOs, 1633386496 bytes 2024-09-14T08:46:50.745 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4000 IOs, 1697869824 bytes 2024-09-14T08:46:51.017 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4100 IOs, 1773981696 bytes 2024-09-14T08:46:51.142 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4200 IOs, 1800446976 bytes 2024-09-14T08:46:51.260 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4300 IOs, 1824601088 bytes 2024-09-14T08:46:51.451 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4400 IOs, 1882226688 bytes 2024-09-14T08:46:51.692 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4500 IOs, 1973617664 bytes 2024-09-14T08:46:51.891 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4600 IOs, 2022545408 bytes 2024-09-14T08:46:52.201 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4700 IOs, 2124536832 bytes 2024-09-14T08:46:52.248 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ error_code=124 2024-09-14T08:46:52.248 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set -e 2024-09-14T08:46:52.248 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T08:46:52.248 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:46:52.248 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T08:46:52.248 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_12 2024-09-14T08:46:52.248 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image_and_enable_mirror cluster2 mirror image_12 snapshot 128M 2024-09-14T08:46:52.248 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:46:52.248 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:46:52.249 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:46:52.249 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:46:52.249 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_12 2024-09-14T08:46:52.249 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:46:52.249 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:46:52.249 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n snapshot ']' 2024-09-14T08:46:52.249 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:46:52.249 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image cluster2 mirror image_12 128M 2024-09-14T08:46:52.249 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:46:52.249 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:46:52.249 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:46:52.249 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:46:52.249 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_12 2024-09-14T08:46:52.250 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:46:52.250 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local size=128 2024-09-14T08:46:52.250 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n 128M ']' 2024-09-14T08:46:52.250 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ size=128M 2024-09-14T08:46:52.250 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:46:52.250 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_12 2024-09-14T08:46:52.296 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T08:46:52.296 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:46:52.296 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' image = image ']' 2024-09-14T08:46:52.296 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:46:52.296 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ enable_mirror cluster2 mirror image_12 snapshot 2024-09-14T08:46:52.296 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:46:52.296 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:46:52.296 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_12 2024-09-14T08:46:52.297 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:46:52.297 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_12 snapshot 2024-09-14T08:46:53.215 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Mirroring enabled 2024-09-14T08:46:53.221 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 info mirror/image_12 2024-09-14T08:46:53.263 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_12': 2024-09-14T08:46:53.263 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T08:46:53.263 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:46:53.263 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 1 2024-09-14T08:46:53.263 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 12e6410e7e38 2024-09-14T08:46:53.263 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.12e6410e7e38 2024-09-14T08:46:53.264 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:46:53.264 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering 2024-09-14T08:46:53.264 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:46:53.264 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:46:53.264 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:46:52 2024 2024-09-14T08:46:53.264 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:46:52 2024 2024-09-14T08:46:53.264 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:46:52 2024 2024-09-14T08:46:53.264 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:46:53.264 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:46:53.264 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: a4a9b638-bea9-4763-b984-2d72d49c1189 2024-09-14T08:46:53.264 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: true 2024-09-14T08:46:53.267 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n '' ']' 2024-09-14T08:46:53.267 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ stress_write_image cluster2 mirror image_12 2024-09-14T08:46:53.267 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:46:53.267 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:46:53.267 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_12 2024-09-14T08:46:53.267 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T08:46:53.269 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local duration=5 2024-09-14T08:46:53.269 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set +e 2024-09-14T08:46:53.269 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ timeout 5s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_12 --debug-rbd=20 --debug-journaler=20 2024-09-14T08:46:53.416 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:100 IOs, 15646720 bytes 2024-09-14T08:46:53.473 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:200 IOs, 16242688 bytes 2024-09-14T08:46:53.607 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:300 IOs, 46930944 bytes 2024-09-14T08:46:53.741 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:400 IOs, 77679616 bytes 2024-09-14T08:46:53.961 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:500 IOs, 169558016 bytes 2024-09-14T08:46:54.218 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:600 IOs, 227167232 bytes 2024-09-14T08:46:54.284 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:700 IOs, 238787584 bytes 2024-09-14T08:46:54.448 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:800 IOs, 276712448 bytes 2024-09-14T08:46:54.579 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:900 IOs, 309843968 bytes 2024-09-14T08:46:54.736 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1000 IOs, 340849664 bytes 2024-09-14T08:46:54.876 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1100 IOs, 375974912 bytes 2024-09-14T08:46:55.031 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1200 IOs, 412602368 bytes 2024-09-14T08:46:55.111 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1300 IOs, 420820992 bytes 2024-09-14T08:46:55.260 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1400 IOs, 457477120 bytes 2024-09-14T08:46:55.372 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1500 IOs, 487299072 bytes 2024-09-14T08:46:55.641 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1600 IOs, 580498432 bytes 2024-09-14T08:46:55.871 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1700 IOs, 628108288 bytes 2024-09-14T08:46:56.159 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1800 IOs, 720549888 bytes 2024-09-14T08:46:56.247 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1900 IOs, 731566080 bytes 2024-09-14T08:46:56.325 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2000 IOs, 738761728 bytes 2024-09-14T08:46:56.400 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2100 IOs, 747694080 bytes 2024-09-14T08:46:56.467 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2200 IOs, 748309504 bytes 2024-09-14T08:46:56.684 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2300 IOs, 812637184 bytes 2024-09-14T08:46:56.810 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2400 IOs, 837895168 bytes 2024-09-14T08:46:56.932 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2500 IOs, 862500864 bytes 2024-09-14T08:46:57.084 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2600 IOs, 906311680 bytes 2024-09-14T08:46:57.268 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2700 IOs, 946088960 bytes 2024-09-14T08:46:57.376 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2800 IOs, 964895744 bytes 2024-09-14T08:46:57.630 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2900 IOs, 1042750464 bytes 2024-09-14T08:46:57.784 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3000 IOs, 1080897536 bytes 2024-09-14T08:46:57.961 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3100 IOs, 1128571904 bytes 2024-09-14T08:46:58.102 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3200 IOs, 1164171264 bytes 2024-09-14T08:46:58.237 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3300 IOs, 1196383232 bytes 2024-09-14T08:46:58.274 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ error_code=124 2024-09-14T08:46:58.275 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set -e 2024-09-14T08:46:58.275 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T08:46:58.275 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:46:58.275 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T08:46:58.275 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_13 2024-09-14T08:46:58.275 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image_and_enable_mirror cluster2 mirror image_13 snapshot 128M 2024-09-14T08:46:58.275 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:46:58.275 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:46:58.275 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:46:58.275 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:46:58.275 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_13 2024-09-14T08:46:58.275 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:46:58.275 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:46:58.276 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n snapshot ']' 2024-09-14T08:46:58.276 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:46:58.276 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image cluster2 mirror image_13 128M 2024-09-14T08:46:58.276 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:46:58.276 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:46:58.276 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:46:58.276 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:46:58.276 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_13 2024-09-14T08:46:58.276 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:46:58.276 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local size=128 2024-09-14T08:46:58.276 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n 128M ']' 2024-09-14T08:46:58.276 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ size=128M 2024-09-14T08:46:58.276 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:46:58.276 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_13 2024-09-14T08:46:58.332 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T08:46:58.332 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:46:58.332 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' image = image ']' 2024-09-14T08:46:58.332 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:46:58.332 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ enable_mirror cluster2 mirror image_13 snapshot 2024-09-14T08:46:58.332 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:46:58.332 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:46:58.332 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_13 2024-09-14T08:46:58.332 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:46:58.333 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_13 snapshot 2024-09-14T08:46:59.218 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Mirroring enabled 2024-09-14T08:46:59.223 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 info mirror/image_13 2024-09-14T08:46:59.263 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_13': 2024-09-14T08:46:59.263 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T08:46:59.263 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:46:59.264 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 1 2024-09-14T08:46:59.264 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 12f22d6111 2024-09-14T08:46:59.264 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.12f22d6111 2024-09-14T08:46:59.264 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:46:59.264 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering 2024-09-14T08:46:59.264 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:46:59.264 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:46:59.264 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:46:58 2024 2024-09-14T08:46:59.264 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:46:58 2024 2024-09-14T08:46:59.264 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:46:58 2024 2024-09-14T08:46:59.264 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:46:59.264 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:46:59.264 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: 5dc0c52c-6377-4410-90b1-49344e6d9458 2024-09-14T08:46:59.264 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: true 2024-09-14T08:46:59.267 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n '' ']' 2024-09-14T08:46:59.267 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ stress_write_image cluster2 mirror image_13 2024-09-14T08:46:59.267 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:46:59.267 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:46:59.267 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_13 2024-09-14T08:46:59.267 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T08:46:59.269 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local duration=12 2024-09-14T08:46:59.269 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set +e 2024-09-14T08:46:59.269 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ timeout 12s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_13 --debug-rbd=20 --debug-journaler=20 2024-09-14T08:46:59.419 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:100 IOs, 23934976 bytes 2024-09-14T08:46:59.546 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:200 IOs, 45449216 bytes 2024-09-14T08:46:59.678 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:300 IOs, 91828224 bytes 2024-09-14T08:46:59.880 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:400 IOs, 149321728 bytes 2024-09-14T08:47:00.078 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:500 IOs, 196488192 bytes 2024-09-14T08:47:00.126 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.9534295094901472) > weight (0.25) 2024-09-14T08:47:00.126 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.1 2024-09-14T08:47:00.126 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Sent signal 15 2024-09-14T08:47:00.127 INFO:tasks.rbd_mirror_thrash:waiting for 2 secs before reviving daemons 2024-09-14T08:47:00.128 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi164.stderr:2024-09-14T08:47:00.127+0000 7f442c8c6640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 (PID: 36798) UID: 1000 2024-09-14T08:47:00.194 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:600 IOs, 226774016 bytes 2024-09-14T08:47:00.386 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:700 IOs, 252256256 bytes 2024-09-14T08:47:00.529 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:800 IOs, 288753664 bytes 2024-09-14T08:47:00.669 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:900 IOs, 319802368 bytes 2024-09-14T08:47:00.828 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1000 IOs, 356410368 bytes 2024-09-14T08:47:01.094 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1100 IOs, 442894336 bytes 2024-09-14T08:47:01.237 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi164.stderr:2024-09-14T08:47:01.236+0000 7fd6dd128640 -1 rbd::mirror::ImageReplayer: 0x55c9fe2c0500 [3/9b43aad5-e6f1-4a31-a25c-d7df22bd01da] register_admin_socket_hook: error registering admin socket commands 2024-09-14T08:47:01.237 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi164.stderr:2024-09-14T08:47:01.237+0000 7fd6dd929640 -1 rbd::mirror::ImageReplayer: 0x55c9fe2c0500 [3/9b43aad5-e6f1-4a31-a25c-d7df22bd01da] register_admin_socket_hook: error registering admin socket commands 2024-09-14T08:47:01.238 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi164.stderr:2024-09-14T08:47:01.237+0000 7f393329b640 -1 rbd::mirror::ImageReplayer: 0x561fbd446500 [3/f2624247-ccb3-494f-a736-10ddf00ce6ae] register_admin_socket_hook: error registering admin socket commands 2024-09-14T08:47:01.238 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi164.stderr:2024-09-14T08:47:01.237+0000 7f3933a9c640 -1 rbd::mirror::ImageReplayer: 0x561fbd446500 [3/f2624247-ccb3-494f-a736-10ddf00ce6ae] register_admin_socket_hook: error registering admin socket commands 2024-09-14T08:47:01.244 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi164.stderr:2024-09-14T08:47:01.244+0000 7fd6dd128640 -1 rbd::mirror::ImageReplayer: 0x55c9fe2c0780 [3/a4a9b638-bea9-4763-b984-2d72d49c1189] register_admin_socket_hook: error registering admin socket commands 2024-09-14T08:47:01.245 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi164.stderr:2024-09-14T08:47:01.244+0000 7fd6dd929640 -1 rbd::mirror::ImageReplayer: 0x55c9fe2c0780 [3/a4a9b638-bea9-4763-b984-2d72d49c1189] register_admin_socket_hook: error registering admin socket commands 2024-09-14T08:47:01.345 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1200 IOs, 497190912 bytes 2024-09-14T08:47:01.490 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1300 IOs, 532222976 bytes 2024-09-14T08:47:01.644 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1400 IOs, 569496576 bytes 2024-09-14T08:47:01.792 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1500 IOs, 614392832 bytes 2024-09-14T08:47:02.055 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1600 IOs, 690232320 bytes 2024-09-14T08:47:02.127 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.1 2024-09-14T08:47:02.128 INFO:teuthology.orchestra.run:waiting for 600 2024-09-14T08:47:02.128 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.1 2024-09-14T08:47:02.128 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Restarting daemon 2024-09-14T08:47:02.128 DEBUG:teuthology.orchestra.run.smithi164:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 2024-09-14T08:47:02.131 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Started 2024-09-14T08:47:02.131 INFO:tasks.rbd_mirror_thrash:waiting for 74 secs before thrashing 2024-09-14T08:47:02.184 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1700 IOs, 711371776 bytes 2024-09-14T08:47:02.322 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1800 IOs, 734396416 bytes 2024-09-14T08:47:02.425 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1900 IOs, 749642752 bytes 2024-09-14T08:47:02.666 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2000 IOs, 822767616 bytes 2024-09-14T08:47:02.941 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2100 IOs, 924194816 bytes 2024-09-14T08:47:03.046 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2200 IOs, 924804096 bytes 2024-09-14T08:47:03.290 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2300 IOs, 1000378368 bytes 2024-09-14T08:47:03.429 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2400 IOs, 1036403712 bytes 2024-09-14T08:47:03.529 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2500 IOs, 1037008896 bytes 2024-09-14T08:47:03.742 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2600 IOs, 1100749824 bytes 2024-09-14T08:47:03.965 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2700 IOs, 1188659200 bytes 2024-09-14T08:47:04.164 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2800 IOs, 1221092352 bytes 2024-09-14T08:47:04.258 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2900 IOs, 1239117824 bytes 2024-09-14T08:47:04.397 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3000 IOs, 1259958272 bytes 2024-09-14T08:47:04.528 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3100 IOs, 1288305664 bytes 2024-09-14T08:47:04.629 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3200 IOs, 1317225472 bytes 2024-09-14T08:47:04.740 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3300 IOs, 1328060416 bytes 2024-09-14T08:47:04.923 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3400 IOs, 1360699392 bytes 2024-09-14T08:47:05.129 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3500 IOs, 1420943360 bytes 2024-09-14T08:47:05.352 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3600 IOs, 1487372288 bytes 2024-09-14T08:47:05.623 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3700 IOs, 1569321984 bytes 2024-09-14T08:47:05.790 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3800 IOs, 1612279808 bytes 2024-09-14T08:47:05.994 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3900 IOs, 1671640064 bytes 2024-09-14T08:47:06.233 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4000 IOs, 1734461440 bytes 2024-09-14T08:47:06.314 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4100 IOs, 1741772800 bytes 2024-09-14T08:47:06.451 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4200 IOs, 1772021760 bytes 2024-09-14T08:47:06.576 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4300 IOs, 1796000768 bytes 2024-09-14T08:47:06.818 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4400 IOs, 1869777920 bytes 2024-09-14T08:47:06.978 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4500 IOs, 1908981760 bytes 2024-09-14T08:47:07.204 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4600 IOs, 1971472384 bytes 2024-09-14T08:47:07.321 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4700 IOs, 1992313856 bytes 2024-09-14T08:47:07.389 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4800 IOs, 1992923136 bytes 2024-09-14T08:47:07.520 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4900 IOs, 2021995520 bytes 2024-09-14T08:47:07.642 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5000 IOs, 2038582272 bytes 2024-09-14T08:47:07.759 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5100 IOs, 2057364480 bytes 2024-09-14T08:47:07.919 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5200 IOs, 2098623488 bytes 2024-09-14T08:47:08.145 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5300 IOs, 2170887168 bytes 2024-09-14T08:47:08.422 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5400 IOs, 2255578112 bytes 2024-09-14T08:47:08.654 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5500 IOs, 2315587584 bytes 2024-09-14T08:47:08.784 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5600 IOs, 2348173312 bytes 2024-09-14T08:47:08.958 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5700 IOs, 2386289664 bytes 2024-09-14T08:47:09.149 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5800 IOs, 2431911936 bytes 2024-09-14T08:47:09.219 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5900 IOs, 2432535552 bytes 2024-09-14T08:47:09.487 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6000 IOs, 2518489088 bytes 2024-09-14T08:47:09.560 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6100 IOs, 2519101440 bytes 2024-09-14T08:47:09.792 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6200 IOs, 2585518080 bytes 2024-09-14T08:47:09.887 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6300 IOs, 2594718720 bytes 2024-09-14T08:47:10.127 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6400 IOs, 2659007488 bytes 2024-09-14T08:47:10.264 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6500 IOs, 2677578752 bytes 2024-09-14T08:47:10.483 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6600 IOs, 2747714560 bytes 2024-09-14T08:47:10.738 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6700 IOs, 2818807808 bytes 2024-09-14T08:47:10.919 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6800 IOs, 2866565120 bytes 2024-09-14T08:47:11.066 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6900 IOs, 2897730560 bytes 2024-09-14T08:47:11.143 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:7000 IOs, 2903297024 bytes 2024-09-14T08:47:11.276 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ error_code=124 2024-09-14T08:47:11.276 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set -e 2024-09-14T08:47:11.276 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T08:47:11.276 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:47:11.276 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T08:47:11.276 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_14 2024-09-14T08:47:11.276 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image_and_enable_mirror cluster2 mirror image_14 snapshot 128M 2024-09-14T08:47:11.276 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:47:11.276 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:47:11.276 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:47:11.277 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:47:11.277 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_14 2024-09-14T08:47:11.277 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:47:11.277 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:47:11.277 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n snapshot ']' 2024-09-14T08:47:11.277 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:47:11.277 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image cluster2 mirror image_14 128M 2024-09-14T08:47:11.277 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:47:11.277 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:47:11.278 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:47:11.278 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:47:11.278 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_14 2024-09-14T08:47:11.278 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:47:11.278 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local size=128 2024-09-14T08:47:11.278 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n 128M ']' 2024-09-14T08:47:11.279 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ size=128M 2024-09-14T08:47:11.279 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:47:11.279 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_14 2024-09-14T08:47:11.323 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T08:47:11.323 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:47:11.323 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' image = image ']' 2024-09-14T08:47:11.323 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:47:11.323 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ enable_mirror cluster2 mirror image_14 snapshot 2024-09-14T08:47:11.323 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:47:11.323 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:47:11.323 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_14 2024-09-14T08:47:11.323 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:47:11.324 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_14 snapshot 2024-09-14T08:47:12.240 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Mirroring enabled 2024-09-14T08:47:12.246 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 info mirror/image_14 2024-09-14T08:47:12.287 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_14': 2024-09-14T08:47:12.287 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T08:47:12.287 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:47:12.287 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 1 2024-09-14T08:47:12.287 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 1304cf5c209b 2024-09-14T08:47:12.288 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.1304cf5c209b 2024-09-14T08:47:12.288 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:47:12.288 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering 2024-09-14T08:47:12.288 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:47:12.288 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:47:12.288 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:47:11 2024 2024-09-14T08:47:12.288 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:47:11 2024 2024-09-14T08:47:12.288 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:47:11 2024 2024-09-14T08:47:12.289 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:47:12.289 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:47:12.289 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: d055c61d-091e-4c03-a847-749c1be38f92 2024-09-14T08:47:12.289 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: true 2024-09-14T08:47:12.290 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n '' ']' 2024-09-14T08:47:12.291 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ stress_write_image cluster2 mirror image_14 2024-09-14T08:47:12.291 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:47:12.291 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:47:12.291 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_14 2024-09-14T08:47:12.291 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T08:47:12.292 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local duration=13 2024-09-14T08:47:12.292 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set +e 2024-09-14T08:47:12.292 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ timeout 13s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_14 --debug-rbd=20 --debug-journaler=20 2024-09-14T08:47:12.595 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:100 IOs, 80681984 bytes 2024-09-14T08:47:12.855 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:200 IOs, 156033024 bytes 2024-09-14T08:47:13.154 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:300 IOs, 282176512 bytes 2024-09-14T08:47:13.394 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:400 IOs, 325362688 bytes 2024-09-14T08:47:13.463 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:500 IOs, 338401280 bytes 2024-09-14T08:47:13.576 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:600 IOs, 348684288 bytes 2024-09-14T08:47:13.717 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:700 IOs, 389313536 bytes 2024-09-14T08:47:13.875 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:800 IOs, 431636480 bytes 2024-09-14T08:47:14.071 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:900 IOs, 480923648 bytes 2024-09-14T08:47:14.268 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1000 IOs, 531883008 bytes 2024-09-14T08:47:14.453 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1100 IOs, 579272704 bytes 2024-09-14T08:47:14.575 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1200 IOs, 599895040 bytes 2024-09-14T08:47:14.705 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1300 IOs, 622696448 bytes 2024-09-14T08:47:14.818 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1400 IOs, 654215168 bytes 2024-09-14T08:47:14.978 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1500 IOs, 682251264 bytes 2024-09-14T08:47:15.093 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1600 IOs, 725403648 bytes 2024-09-14T08:47:15.296 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1700 IOs, 759543808 bytes 2024-09-14T08:47:15.363 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1800 IOs, 760122368 bytes 2024-09-14T08:47:15.473 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1900 IOs, 783275008 bytes 2024-09-14T08:47:15.695 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2000 IOs, 843303936 bytes 2024-09-14T08:47:15.913 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2100 IOs, 899812352 bytes 2024-09-14T08:47:15.982 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2200 IOs, 901459968 bytes 2024-09-14T08:47:16.048 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2300 IOs, 909913088 bytes 2024-09-14T08:47:16.177 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2400 IOs, 935682048 bytes 2024-09-14T08:47:16.349 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2500 IOs, 983873536 bytes 2024-09-14T08:47:16.560 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2600 IOs, 1032546304 bytes 2024-09-14T08:47:16.714 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2700 IOs, 1068491776 bytes 2024-09-14T08:47:16.953 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2800 IOs, 1149365248 bytes 2024-09-14T08:47:17.195 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2900 IOs, 1209798656 bytes 2024-09-14T08:47:17.342 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3000 IOs, 1244032000 bytes 2024-09-14T08:47:17.418 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3100 IOs, 1252396032 bytes 2024-09-14T08:47:17.640 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3200 IOs, 1315800064 bytes 2024-09-14T08:47:17.712 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3300 IOs, 1316420608 bytes 2024-09-14T08:47:17.900 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3400 IOs, 1367250944 bytes 2024-09-14T08:47:18.039 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3500 IOs, 1396837376 bytes 2024-09-14T08:47:18.252 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3600 IOs, 1458363392 bytes 2024-09-14T08:47:18.497 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3700 IOs, 1531141120 bytes 2024-09-14T08:47:18.627 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3800 IOs, 1556652032 bytes 2024-09-14T08:47:18.832 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3900 IOs, 1616717824 bytes 2024-09-14T08:47:18.973 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4000 IOs, 1640971264 bytes 2024-09-14T08:47:19.038 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4100 IOs, 1641824256 bytes 2024-09-14T08:47:19.285 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4200 IOs, 1722536960 bytes 2024-09-14T08:47:19.429 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4300 IOs, 1755368448 bytes 2024-09-14T08:47:19.612 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4400 IOs, 1796262912 bytes 2024-09-14T08:47:19.736 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4500 IOs, 1846516736 bytes 2024-09-14T08:47:20.047 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4600 IOs, 1931682816 bytes 2024-09-14T08:47:20.289 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4700 IOs, 2018767872 bytes 2024-09-14T08:47:20.627 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4800 IOs, 2108290048 bytes 2024-09-14T08:47:20.763 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4900 IOs, 2146667520 bytes 2024-09-14T08:47:20.933 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5000 IOs, 2199099392 bytes 2024-09-14T08:47:21.103 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5100 IOs, 2227747840 bytes 2024-09-14T08:47:21.187 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5200 IOs, 2233865216 bytes 2024-09-14T08:47:21.322 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5300 IOs, 2261165056 bytes 2024-09-14T08:47:21.460 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5400 IOs, 2289180672 bytes 2024-09-14T08:47:21.537 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5500 IOs, 2304837632 bytes 2024-09-14T08:47:21.699 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5600 IOs, 2340066304 bytes 2024-09-14T08:47:21.784 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5700 IOs, 2340688896 bytes 2024-09-14T08:47:21.861 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5800 IOs, 2345109504 bytes 2024-09-14T08:47:21.988 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5900 IOs, 2369149952 bytes 2024-09-14T08:47:22.104 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6000 IOs, 2394735616 bytes 2024-09-14T08:47:22.434 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6100 IOs, 2513478656 bytes 2024-09-14T08:47:22.669 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6200 IOs, 2562545664 bytes 2024-09-14T08:47:22.841 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6300 IOs, 2604615680 bytes 2024-09-14T08:47:23.031 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6400 IOs, 2669009920 bytes 2024-09-14T08:47:23.303 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6500 IOs, 2738838528 bytes 2024-09-14T08:47:23.600 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6600 IOs, 2836790272 bytes 2024-09-14T08:47:23.681 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6700 IOs, 2837397504 bytes 2024-09-14T08:47:23.766 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6800 IOs, 2863909888 bytes 2024-09-14T08:47:23.907 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.0 2024-09-14T08:47:23.908 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Sent signal 15 2024-09-14T08:47:23.908 INFO:tasks.rbd_mirror_thrash:waiting for 7 secs before reviving daemons 2024-09-14T08:47:23.909 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi164.stderr:2024-09-14T08:47:23.909+0000 7fe3a5f0d640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 (PID: 39941) UID: 1000 2024-09-14T08:47:23.970 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6900 IOs, 2896594944 bytes 2024-09-14T08:47:24.198 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:7000 IOs, 2962838528 bytes 2024-09-14T08:47:24.307 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:7100 IOs, 2977559552 bytes 2024-09-14T08:47:24.451 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:7200 IOs, 3016404992 bytes 2024-09-14T08:47:24.539 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:7300 IOs, 3022598144 bytes 2024-09-14T08:47:24.631 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:7400 IOs, 3030429696 bytes 2024-09-14T08:47:24.724 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:7500 IOs, 3041215488 bytes 2024-09-14T08:47:24.962 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:7600 IOs, 3107276800 bytes 2024-09-14T08:47:25.123 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:7700 IOs, 3147029504 bytes 2024-09-14T08:47:25.271 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:7800 IOs, 3185012736 bytes 2024-09-14T08:47:25.301 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ error_code=124 2024-09-14T08:47:25.301 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set -e 2024-09-14T08:47:25.301 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T08:47:25.301 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:47:25.301 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T08:47:25.301 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_15 2024-09-14T08:47:25.301 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image_and_enable_mirror cluster2 mirror image_15 snapshot 128M 2024-09-14T08:47:25.301 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:47:25.301 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:47:25.301 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:47:25.301 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:47:25.302 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_15 2024-09-14T08:47:25.302 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:47:25.302 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:47:25.302 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n snapshot ']' 2024-09-14T08:47:25.302 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:47:25.302 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image cluster2 mirror image_15 128M 2024-09-14T08:47:25.302 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:47:25.302 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:47:25.302 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:47:25.302 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:47:25.302 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_15 2024-09-14T08:47:25.302 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:47:25.302 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local size=128 2024-09-14T08:47:25.302 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n 128M ']' 2024-09-14T08:47:25.302 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ size=128M 2024-09-14T08:47:25.303 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:47:25.303 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_15 2024-09-14T08:47:25.361 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T08:47:25.361 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:47:25.361 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' image = image ']' 2024-09-14T08:47:25.361 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:47:25.361 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ enable_mirror cluster2 mirror image_15 snapshot 2024-09-14T08:47:25.361 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:47:25.361 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:47:25.361 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_15 2024-09-14T08:47:25.361 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:47:25.361 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_15 snapshot 2024-09-14T08:47:26.261 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Mirroring enabled 2024-09-14T08:47:26.267 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 info mirror/image_15 2024-09-14T08:47:26.311 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_15': 2024-09-14T08:47:26.311 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T08:47:26.311 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:47:26.311 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 1 2024-09-14T08:47:26.311 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 13105a110027 2024-09-14T08:47:26.311 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.13105a110027 2024-09-14T08:47:26.312 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:47:26.312 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering 2024-09-14T08:47:26.312 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:47:26.312 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:47:26.312 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:47:25 2024 2024-09-14T08:47:26.312 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:47:25 2024 2024-09-14T08:47:26.312 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:47:25 2024 2024-09-14T08:47:26.312 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:47:26.312 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:47:26.312 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: b5a7a156-c2fc-47f1-b818-af6739582d53 2024-09-14T08:47:26.312 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: true 2024-09-14T08:47:26.315 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n '' ']' 2024-09-14T08:47:26.315 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ stress_write_image cluster2 mirror image_15 2024-09-14T08:47:26.315 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:47:26.315 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:47:26.315 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_15 2024-09-14T08:47:26.315 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T08:47:26.317 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local duration=9 2024-09-14T08:47:26.317 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set +e 2024-09-14T08:47:26.317 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ timeout 9s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_15 --debug-rbd=20 --debug-journaler=20 2024-09-14T08:47:26.690 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:100 IOs, 138232832 bytes 2024-09-14T08:47:26.984 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:200 IOs, 217543680 bytes 2024-09-14T08:47:27.163 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:300 IOs, 259491840 bytes 2024-09-14T08:47:27.288 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:400 IOs, 282335232 bytes 2024-09-14T08:47:27.373 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:500 IOs, 293373952 bytes 2024-09-14T08:47:27.605 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:600 IOs, 377431040 bytes 2024-09-14T08:47:27.805 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:700 IOs, 419862528 bytes 2024-09-14T08:47:28.002 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:800 IOs, 468776960 bytes 2024-09-14T08:47:28.105 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:900 IOs, 486044672 bytes 2024-09-14T08:47:28.431 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1000 IOs, 594359296 bytes 2024-09-14T08:47:28.609 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1100 IOs, 641585152 bytes 2024-09-14T08:47:28.782 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1200 IOs, 689888256 bytes 2024-09-14T08:47:28.971 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1300 IOs, 725957632 bytes 2024-09-14T08:47:29.097 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1400 IOs, 752734208 bytes 2024-09-14T08:47:29.221 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1500 IOs, 789226496 bytes 2024-09-14T08:47:29.316 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1600 IOs, 789836800 bytes 2024-09-14T08:47:29.464 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1700 IOs, 826710016 bytes 2024-09-14T08:47:29.778 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1800 IOs, 942898176 bytes 2024-09-14T08:47:30.088 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1900 IOs, 961230848 bytes 2024-09-14T08:47:30.325 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2000 IOs, 1035630592 bytes 2024-09-14T08:47:30.432 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2100 IOs, 1048268800 bytes 2024-09-14T08:47:30.711 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2200 IOs, 1139208192 bytes 2024-09-14T08:47:30.773 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2300 IOs, 1142515712 bytes 2024-09-14T08:47:30.909 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.0 2024-09-14T08:47:30.909 INFO:teuthology.orchestra.run:waiting for 600 2024-09-14T08:47:30.909 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.0 2024-09-14T08:47:30.909 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Restarting daemon 2024-09-14T08:47:30.910 DEBUG:teuthology.orchestra.run.smithi164:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 2024-09-14T08:47:30.915 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Started 2024-09-14T08:47:30.916 INFO:tasks.rbd_mirror_thrash:waiting for 70 secs before thrashing 2024-09-14T08:47:30.929 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2400 IOs, 1183399936 bytes 2024-09-14T08:47:31.135 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2500 IOs, 1238260736 bytes 2024-09-14T08:47:31.296 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2600 IOs, 1266216960 bytes 2024-09-14T08:47:31.550 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2700 IOs, 1337239552 bytes 2024-09-14T08:47:31.751 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2800 IOs, 1418511360 bytes 2024-09-14T08:47:31.883 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2900 IOs, 1421181952 bytes 2024-09-14T08:47:32.172 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3000 IOs, 1511508992 bytes 2024-09-14T08:47:32.242 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3100 IOs, 1514844160 bytes 2024-09-14T08:47:32.366 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3200 IOs, 1565102080 bytes 2024-09-14T08:47:32.616 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3300 IOs, 1618406400 bytes 2024-09-14T08:47:32.711 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3400 IOs, 1639391232 bytes 2024-09-14T08:47:32.882 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3500 IOs, 1670075392 bytes 2024-09-14T08:47:33.094 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3600 IOs, 1728200704 bytes 2024-09-14T08:47:33.360 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3700 IOs, 1813776384 bytes 2024-09-14T08:47:33.558 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3800 IOs, 1867770880 bytes 2024-09-14T08:47:33.711 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3900 IOs, 1901694976 bytes 2024-09-14T08:47:33.917 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4000 IOs, 1960003584 bytes 2024-09-14T08:47:34.118 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4100 IOs, 2022299648 bytes 2024-09-14T08:47:34.283 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4200 IOs, 2051566592 bytes 2024-09-14T08:47:34.381 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4300 IOs, 2064492544 bytes 2024-09-14T08:47:34.506 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4400 IOs, 2089561088 bytes 2024-09-14T08:47:34.623 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4500 IOs, 2112678912 bytes 2024-09-14T08:47:34.793 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4600 IOs, 2157630464 bytes 2024-09-14T08:47:34.871 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4700 IOs, 2158519296 bytes 2024-09-14T08:47:35.075 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4800 IOs, 2233395200 bytes 2024-09-14T08:47:35.252 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4900 IOs, 2253097984 bytes 2024-09-14T08:47:35.323 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ error_code=124 2024-09-14T08:47:35.323 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set -e 2024-09-14T08:47:35.323 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T08:47:35.323 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:47:35.323 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T08:47:35.323 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_16 2024-09-14T08:47:35.323 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image_and_enable_mirror cluster2 mirror image_16 snapshot 128M 2024-09-14T08:47:35.323 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:47:35.323 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:47:35.323 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:47:35.324 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:47:35.324 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_16 2024-09-14T08:47:35.324 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:47:35.324 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:47:35.324 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n snapshot ']' 2024-09-14T08:47:35.324 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:47:35.324 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image cluster2 mirror image_16 128M 2024-09-14T08:47:35.324 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:47:35.324 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:47:35.324 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:47:35.324 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:47:35.324 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_16 2024-09-14T08:47:35.324 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:47:35.324 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local size=128 2024-09-14T08:47:35.325 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n 128M ']' 2024-09-14T08:47:35.325 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ size=128M 2024-09-14T08:47:35.325 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:47:35.325 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_16 2024-09-14T08:47:35.385 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T08:47:35.386 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:47:35.386 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' image = image ']' 2024-09-14T08:47:35.386 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:47:35.386 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ enable_mirror cluster2 mirror image_16 snapshot 2024-09-14T08:47:35.386 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:47:35.386 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:47:35.386 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_16 2024-09-14T08:47:35.386 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:47:35.386 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_16 snapshot 2024-09-14T08:47:36.273 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Mirroring enabled 2024-09-14T08:47:36.279 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 info mirror/image_16 2024-09-14T08:47:36.320 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_16': 2024-09-14T08:47:36.321 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T08:47:36.321 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:47:36.321 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 1 2024-09-14T08:47:36.321 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 13267dba0b6c 2024-09-14T08:47:36.321 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.13267dba0b6c 2024-09-14T08:47:36.321 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:47:36.321 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering 2024-09-14T08:47:36.321 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:47:36.321 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:47:36.321 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:47:35 2024 2024-09-14T08:47:36.321 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:47:35 2024 2024-09-14T08:47:36.321 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:47:35 2024 2024-09-14T08:47:36.321 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:47:36.322 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:47:36.322 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: 095f96c4-d4b7-4fe2-8ba2-0b31704d281f 2024-09-14T08:47:36.322 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: true 2024-09-14T08:47:36.323 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n '' ']' 2024-09-14T08:47:36.324 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ stress_write_image cluster2 mirror image_16 2024-09-14T08:47:36.324 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:47:36.324 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:47:36.324 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_16 2024-09-14T08:47:36.324 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T08:47:36.325 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local duration=8 2024-09-14T08:47:36.325 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set +e 2024-09-14T08:47:36.325 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ timeout 8s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_16 --debug-rbd=20 --debug-journaler=20 2024-09-14T08:47:36.537 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:100 IOs, 45919232 bytes 2024-09-14T08:47:36.753 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:200 IOs, 107637760 bytes 2024-09-14T08:47:36.813 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:300 IOs, 108229632 bytes 2024-09-14T08:47:36.930 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:400 IOs, 131283968 bytes 2024-09-14T08:47:37.099 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:500 IOs, 181678080 bytes 2024-09-14T08:47:37.374 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:600 IOs, 257554432 bytes 2024-09-14T08:47:37.545 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:700 IOs, 295308288 bytes 2024-09-14T08:47:37.633 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:800 IOs, 314621952 bytes 2024-09-14T08:47:37.802 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:900 IOs, 355517440 bytes 2024-09-14T08:47:38.000 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1000 IOs, 398426112 bytes 2024-09-14T08:47:38.168 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1100 IOs, 438920192 bytes 2024-09-14T08:47:38.318 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1200 IOs, 474889216 bytes 2024-09-14T08:47:38.470 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1300 IOs, 513752064 bytes 2024-09-14T08:47:38.691 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1400 IOs, 578493440 bytes 2024-09-14T08:47:38.854 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1500 IOs, 614051840 bytes 2024-09-14T08:47:39.052 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1600 IOs, 672427008 bytes 2024-09-14T08:47:39.222 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1700 IOs, 702136320 bytes 2024-09-14T08:47:39.436 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1800 IOs, 785411072 bytes 2024-09-14T08:47:39.577 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1900 IOs, 793767936 bytes 2024-09-14T08:47:39.734 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2000 IOs, 832800768 bytes 2024-09-14T08:47:39.840 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2100 IOs, 848505856 bytes 2024-09-14T08:47:39.996 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2200 IOs, 887431168 bytes 2024-09-14T08:47:40.091 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2300 IOs, 904643584 bytes 2024-09-14T08:47:40.199 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2400 IOs, 942361600 bytes 2024-09-14T08:47:40.365 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2500 IOs, 964686848 bytes 2024-09-14T08:47:40.510 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2600 IOs, 995704832 bytes 2024-09-14T08:47:40.639 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2700 IOs, 1033798656 bytes 2024-09-14T08:47:40.901 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2800 IOs, 1102465024 bytes 2024-09-14T08:47:41.034 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2900 IOs, 1131711488 bytes 2024-09-14T08:47:41.226 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3000 IOs, 1203709952 bytes 2024-09-14T08:47:41.407 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3100 IOs, 1258807296 bytes 2024-09-14T08:47:41.672 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3200 IOs, 1312594944 bytes 2024-09-14T08:47:41.960 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3300 IOs, 1412085760 bytes 2024-09-14T08:47:42.054 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3400 IOs, 1412685824 bytes 2024-09-14T08:47:42.272 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3500 IOs, 1475534848 bytes 2024-09-14T08:47:42.439 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3600 IOs, 1517164544 bytes 2024-09-14T08:47:42.669 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3700 IOs, 1567752192 bytes 2024-09-14T08:47:42.916 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3800 IOs, 1643061248 bytes 2024-09-14T08:47:43.094 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3900 IOs, 1689000960 bytes 2024-09-14T08:47:43.338 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4000 IOs, 1782298624 bytes 2024-09-14T08:47:43.582 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4100 IOs, 1835225088 bytes 2024-09-14T08:47:43.651 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4200 IOs, 1835842560 bytes 2024-09-14T08:47:43.779 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4300 IOs, 1860248576 bytes 2024-09-14T08:47:43.958 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4400 IOs, 1915756544 bytes 2024-09-14T08:47:44.183 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4500 IOs, 1995270144 bytes 2024-09-14T08:47:44.335 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ error_code=124 2024-09-14T08:47:44.335 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set -e 2024-09-14T08:47:44.335 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T08:47:44.335 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:47:44.335 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T08:47:44.335 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_17 2024-09-14T08:47:44.335 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image_and_enable_mirror cluster2 mirror image_17 snapshot 128M 2024-09-14T08:47:44.335 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:47:44.335 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:47:44.336 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:47:44.336 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:47:44.336 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_17 2024-09-14T08:47:44.336 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:47:44.336 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:47:44.336 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n snapshot ']' 2024-09-14T08:47:44.336 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:47:44.336 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image cluster2 mirror image_17 128M 2024-09-14T08:47:44.336 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:47:44.336 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:47:44.336 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:47:44.336 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:47:44.336 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_17 2024-09-14T08:47:44.336 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:47:44.337 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local size=128 2024-09-14T08:47:44.337 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n 128M ']' 2024-09-14T08:47:44.337 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ size=128M 2024-09-14T08:47:44.337 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:47:44.337 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_17 2024-09-14T08:47:44.392 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T08:47:44.392 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:47:44.392 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' image = image ']' 2024-09-14T08:47:44.392 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:47:44.392 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ enable_mirror cluster2 mirror image_17 snapshot 2024-09-14T08:47:44.393 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:47:44.393 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:47:44.393 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_17 2024-09-14T08:47:44.393 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:47:44.393 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_17 snapshot 2024-09-14T08:47:45.294 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Mirroring enabled 2024-09-14T08:47:45.300 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 info mirror/image_17 2024-09-14T08:47:45.343 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_17': 2024-09-14T08:47:45.343 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T08:47:45.343 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:47:45.343 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 1 2024-09-14T08:47:45.344 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 133287188709 2024-09-14T08:47:45.344 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.133287188709 2024-09-14T08:47:45.344 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:47:45.344 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering 2024-09-14T08:47:45.344 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:47:45.344 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:47:45.344 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:47:44 2024 2024-09-14T08:47:45.344 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:47:44 2024 2024-09-14T08:47:45.344 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:47:44 2024 2024-09-14T08:47:45.344 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:47:45.344 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:47:45.344 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: e90f529e-8f18-40e1-aa77-39aa510ca09b 2024-09-14T08:47:45.345 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: true 2024-09-14T08:47:45.347 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n '' ']' 2024-09-14T08:47:45.347 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ stress_write_image cluster2 mirror image_17 2024-09-14T08:47:45.347 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:47:45.347 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:47:45.347 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_17 2024-09-14T08:47:45.347 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T08:47:45.349 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local duration=13 2024-09-14T08:47:45.349 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set +e 2024-09-14T08:47:45.349 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ timeout 13s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_17 --debug-rbd=20 --debug-journaler=20 2024-09-14T08:47:45.487 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:100 IOs, 10482688 bytes 2024-09-14T08:47:45.724 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:200 IOs, 60469248 bytes 2024-09-14T08:47:46.055 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:300 IOs, 168823808 bytes 2024-09-14T08:47:46.310 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:400 IOs, 244850688 bytes 2024-09-14T08:47:46.405 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:500 IOs, 256432128 bytes 2024-09-14T08:47:46.484 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:600 IOs, 265227264 bytes 2024-09-14T08:47:46.620 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:700 IOs, 291420160 bytes 2024-09-14T08:47:46.800 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:800 IOs, 319866880 bytes 2024-09-14T08:47:46.960 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:900 IOs, 356743168 bytes 2024-09-14T08:47:47.125 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1000 IOs, 393665536 bytes 2024-09-14T08:47:47.345 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1100 IOs, 461477888 bytes 2024-09-14T08:47:47.538 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1200 IOs, 519980032 bytes 2024-09-14T08:47:47.713 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1300 IOs, 557409280 bytes 2024-09-14T08:47:47.923 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1400 IOs, 622027776 bytes 2024-09-14T08:47:48.170 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1500 IOs, 690240512 bytes 2024-09-14T08:47:48.435 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1600 IOs, 762996736 bytes 2024-09-14T08:47:48.753 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1700 IOs, 867614720 bytes 2024-09-14T08:47:49.049 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1800 IOs, 978907136 bytes 2024-09-14T08:47:49.231 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1900 IOs, 1002917888 bytes 2024-09-14T08:47:49.381 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2000 IOs, 1034929152 bytes 2024-09-14T08:47:49.616 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2100 IOs, 1116809216 bytes 2024-09-14T08:47:49.751 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2200 IOs, 1131702272 bytes 2024-09-14T08:47:49.994 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2300 IOs, 1209235456 bytes 2024-09-14T08:47:50.221 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2400 IOs, 1271964672 bytes 2024-09-14T08:47:50.294 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2500 IOs, 1275018240 bytes 2024-09-14T08:47:50.429 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2600 IOs, 1309341696 bytes 2024-09-14T08:47:50.584 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2700 IOs, 1329400832 bytes 2024-09-14T08:47:50.778 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2800 IOs, 1387521024 bytes 2024-09-14T08:47:50.918 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2900 IOs, 1411920896 bytes 2024-09-14T08:47:51.042 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3000 IOs, 1434289152 bytes 2024-09-14T08:47:51.215 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3100 IOs, 1479088128 bytes 2024-09-14T08:47:51.282 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3200 IOs, 1479693312 bytes 2024-09-14T08:47:51.383 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3300 IOs, 1496625152 bytes 2024-09-14T08:47:51.525 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3400 IOs, 1521780736 bytes 2024-09-14T08:47:51.606 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3500 IOs, 1532199936 bytes 2024-09-14T08:47:51.757 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3600 IOs, 1561755648 bytes 2024-09-14T08:47:51.952 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3700 IOs, 1615597568 bytes 2024-09-14T08:47:52.147 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3800 IOs, 1669504000 bytes 2024-09-14T08:47:52.394 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3900 IOs, 1740313600 bytes 2024-09-14T08:47:52.562 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4000 IOs, 1795057664 bytes 2024-09-14T08:47:52.765 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4100 IOs, 1831108608 bytes 2024-09-14T08:47:52.901 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4200 IOs, 1868618752 bytes 2024-09-14T08:47:53.076 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4300 IOs, 1903783936 bytes 2024-09-14T08:47:53.196 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4400 IOs, 1926100992 bytes 2024-09-14T08:47:53.387 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4500 IOs, 1988090880 bytes 2024-09-14T08:47:53.528 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4600 IOs, 2003450880 bytes 2024-09-14T08:47:53.652 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4700 IOs, 2031380480 bytes 2024-09-14T08:47:53.885 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4800 IOs, 2099489792 bytes 2024-09-14T08:47:53.994 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4900 IOs, 2108284928 bytes 2024-09-14T08:47:54.120 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5000 IOs, 2131130368 bytes 2024-09-14T08:47:54.262 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5100 IOs, 2201920512 bytes 2024-09-14T08:47:54.551 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5200 IOs, 2245536768 bytes 2024-09-14T08:47:54.679 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5300 IOs, 2269554688 bytes 2024-09-14T08:47:54.838 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5400 IOs, 2317155328 bytes 2024-09-14T08:47:54.989 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5500 IOs, 2338283520 bytes 2024-09-14T08:47:55.120 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5600 IOs, 2364303360 bytes 2024-09-14T08:47:55.318 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5700 IOs, 2416440320 bytes 2024-09-14T08:47:55.442 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5800 IOs, 2438046720 bytes 2024-09-14T08:47:55.651 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5900 IOs, 2495112192 bytes 2024-09-14T08:47:55.760 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6000 IOs, 2512954368 bytes 2024-09-14T08:47:55.972 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6100 IOs, 2572128256 bytes 2024-09-14T08:47:56.084 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6200 IOs, 2588088320 bytes 2024-09-14T08:47:56.310 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6300 IOs, 2659139584 bytes 2024-09-14T08:47:56.433 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6400 IOs, 2674969600 bytes 2024-09-14T08:47:56.755 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6500 IOs, 2786338816 bytes 2024-09-14T08:47:56.904 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6600 IOs, 2837010432 bytes 2024-09-14T08:47:57.115 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6700 IOs, 2870797312 bytes 2024-09-14T08:47:57.242 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6800 IOs, 2898161664 bytes 2024-09-14T08:47:57.358 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6900 IOs, 2911858688 bytes 2024-09-14T08:47:57.567 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:7000 IOs, 2971678720 bytes 2024-09-14T08:47:57.764 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:7100 IOs, 3027001344 bytes 2024-09-14T08:47:57.971 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:7200 IOs, 3075031040 bytes 2024-09-14T08:47:58.277 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:7300 IOs, 3163098112 bytes 2024-09-14T08:47:58.357 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ error_code=124 2024-09-14T08:47:58.357 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set -e 2024-09-14T08:47:58.357 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T08:47:58.357 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:47:58.357 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T08:47:58.357 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_18 2024-09-14T08:47:58.357 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image_and_enable_mirror cluster2 mirror image_18 snapshot 128M 2024-09-14T08:47:58.357 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:47:58.357 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:47:58.357 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:47:58.357 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:47:58.358 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_18 2024-09-14T08:47:58.358 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:47:58.358 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:47:58.358 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n snapshot ']' 2024-09-14T08:47:58.358 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:47:58.358 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image cluster2 mirror image_18 128M 2024-09-14T08:47:58.358 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:47:58.358 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:47:58.358 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:47:58.358 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:47:58.358 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_18 2024-09-14T08:47:58.359 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:47:58.359 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local size=128 2024-09-14T08:47:58.359 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n 128M ']' 2024-09-14T08:47:58.359 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ size=128M 2024-09-14T08:47:58.359 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:47:58.360 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_18 2024-09-14T08:47:58.608 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T08:47:58.608 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:47:58.608 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' image = image ']' 2024-09-14T08:47:58.608 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:47:58.608 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ enable_mirror cluster2 mirror image_18 snapshot 2024-09-14T08:47:58.608 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:47:58.609 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:47:58.609 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_18 2024-09-14T08:47:58.609 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:47:58.609 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_18 snapshot 2024-09-14T08:47:59.304 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Mirroring enabled 2024-09-14T08:47:59.309 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 info mirror/image_18 2024-09-14T08:47:59.349 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_18': 2024-09-14T08:47:59.349 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T08:47:59.349 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:47:59.349 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 1 2024-09-14T08:47:59.349 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 133ee2b97dbb 2024-09-14T08:47:59.349 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.133ee2b97dbb 2024-09-14T08:47:59.349 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:47:59.349 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering 2024-09-14T08:47:59.350 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:47:59.350 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:47:59.350 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:47:58 2024 2024-09-14T08:47:59.350 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:47:58 2024 2024-09-14T08:47:59.350 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:47:58 2024 2024-09-14T08:47:59.350 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:47:59.350 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:47:59.350 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: 14306d77-7330-41b5-b70c-85c15b66b18d 2024-09-14T08:47:59.350 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: true 2024-09-14T08:47:59.352 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n '' ']' 2024-09-14T08:47:59.352 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ stress_write_image cluster2 mirror image_18 2024-09-14T08:47:59.353 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:47:59.353 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:47:59.353 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_18 2024-09-14T08:47:59.353 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T08:47:59.354 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local duration=10 2024-09-14T08:47:59.354 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set +e 2024-09-14T08:47:59.354 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ timeout 10s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_18 --debug-rbd=20 --debug-journaler=20 2024-09-14T08:47:59.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:100 IOs, 57037824 bytes 2024-09-14T08:47:59.841 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:200 IOs, 133052416 bytes 2024-09-14T08:47:59.934 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:300 IOs, 138184704 bytes 2024-09-14T08:48:00.146 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:400 IOs, 197016576 bytes 2024-09-14T08:48:00.295 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:500 IOs, 231432192 bytes 2024-09-14T08:48:00.409 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:600 IOs, 248781824 bytes 2024-09-14T08:48:00.508 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:700 IOs, 272988160 bytes 2024-09-14T08:48:00.710 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:800 IOs, 315054080 bytes 2024-09-14T08:48:00.779 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:900 IOs, 315692032 bytes 2024-09-14T08:48:00.901 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1000 IOs, 334884864 bytes 2024-09-14T08:48:01.131 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1100 IOs, 394864640 bytes 2024-09-14T08:48:01.291 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1200 IOs, 418445312 bytes 2024-09-14T08:48:01.551 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1300 IOs, 499192832 bytes 2024-09-14T08:48:01.717 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1400 IOs, 548182016 bytes 2024-09-14T08:48:01.829 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1500 IOs, 553476096 bytes 2024-09-14T08:48:02.030 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1600 IOs, 605842432 bytes 2024-09-14T08:48:02.219 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1700 IOs, 651517952 bytes 2024-09-14T08:48:02.371 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1800 IOs, 689818624 bytes 2024-09-14T08:48:02.512 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1900 IOs, 714343424 bytes 2024-09-14T08:48:02.640 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2000 IOs, 741288960 bytes 2024-09-14T08:48:02.767 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2100 IOs, 774539264 bytes 2024-09-14T08:48:02.949 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2200 IOs, 802833408 bytes 2024-09-14T08:48:03.022 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2300 IOs, 803457024 bytes 2024-09-14T08:48:03.093 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2400 IOs, 804099072 bytes 2024-09-14T08:48:03.239 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2500 IOs, 831676416 bytes 2024-09-14T08:48:03.526 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2600 IOs, 918677504 bytes 2024-09-14T08:48:03.663 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2700 IOs, 954468352 bytes 2024-09-14T08:48:03.933 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2800 IOs, 1029437440 bytes 2024-09-14T08:48:04.084 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2900 IOs, 1070390272 bytes 2024-09-14T08:48:04.230 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3000 IOs, 1098233856 bytes 2024-09-14T08:48:04.319 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3100 IOs, 1104084992 bytes 2024-09-14T08:48:04.420 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3200 IOs, 1117795328 bytes 2024-09-14T08:48:04.669 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3300 IOs, 1188210688 bytes 2024-09-14T08:48:04.826 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3400 IOs, 1241589760 bytes 2024-09-14T08:48:04.958 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3500 IOs, 1245980672 bytes 2024-09-14T08:48:05.101 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3600 IOs, 1282861056 bytes 2024-09-14T08:48:05.200 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3700 IOs, 1283471360 bytes 2024-09-14T08:48:05.308 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3800 IOs, 1295643648 bytes 2024-09-14T08:48:05.445 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3900 IOs, 1321211904 bytes 2024-09-14T08:48:05.643 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4000 IOs, 1373592576 bytes 2024-09-14T08:48:05.783 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4100 IOs, 1399291904 bytes 2024-09-14T08:48:05.951 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4200 IOs, 1441085440 bytes 2024-09-14T08:48:06.088 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4300 IOs, 1491710976 bytes 2024-09-14T08:48:06.249 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4400 IOs, 1538576384 bytes 2024-09-14T08:48:06.550 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4500 IOs, 1620814848 bytes 2024-09-14T08:48:06.687 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4600 IOs, 1636402176 bytes 2024-09-14T08:48:06.970 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4700 IOs, 1719404544 bytes 2024-09-14T08:48:07.163 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4800 IOs, 1770067968 bytes 2024-09-14T08:48:07.422 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4900 IOs, 1845265408 bytes 2024-09-14T08:48:07.553 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5000 IOs, 1886433280 bytes 2024-09-14T08:48:07.826 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5100 IOs, 1955048448 bytes 2024-09-14T08:48:07.927 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5200 IOs, 1965682688 bytes 2024-09-14T08:48:08.142 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5300 IOs, 2021913600 bytes 2024-09-14T08:48:08.265 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5400 IOs, 2042164224 bytes 2024-09-14T08:48:08.435 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5500 IOs, 2078982144 bytes 2024-09-14T08:48:08.507 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5600 IOs, 2079580160 bytes 2024-09-14T08:48:08.667 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5700 IOs, 2113903616 bytes 2024-09-14T08:48:08.764 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5800 IOs, 2122009600 bytes 2024-09-14T08:48:08.909 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5900 IOs, 2152516608 bytes 2024-09-14T08:48:09.078 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6000 IOs, 2188444672 bytes 2024-09-14T08:48:09.336 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6100 IOs, 2263296000 bytes 2024-09-14T08:48:09.361 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ error_code=124 2024-09-14T08:48:09.361 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set -e 2024-09-14T08:48:09.361 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T08:48:09.361 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:48:09.361 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T08:48:09.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_19 2024-09-14T08:48:09.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image_and_enable_mirror cluster2 mirror image_19 snapshot 128M 2024-09-14T08:48:09.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:48:09.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:48:09.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:48:09.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:48:09.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_19 2024-09-14T08:48:09.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:48:09.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:48:09.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n snapshot ']' 2024-09-14T08:48:09.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:48:09.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image cluster2 mirror image_19 128M 2024-09-14T08:48:09.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:48:09.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:48:09.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:48:09.363 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:48:09.363 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_19 2024-09-14T08:48:09.363 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:48:09.363 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local size=128 2024-09-14T08:48:09.363 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n 128M ']' 2024-09-14T08:48:09.363 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ size=128M 2024-09-14T08:48:09.363 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:48:09.363 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_19 2024-09-14T08:48:09.410 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T08:48:09.410 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:48:09.410 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' image = image ']' 2024-09-14T08:48:09.410 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:48:09.410 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ enable_mirror cluster2 mirror image_19 snapshot 2024-09-14T08:48:09.410 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:48:09.410 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:48:09.410 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_19 2024-09-14T08:48:09.410 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:48:09.411 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_19 snapshot 2024-09-14T08:48:10.316 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Mirroring enabled 2024-09-14T08:48:10.321 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 info mirror/image_19 2024-09-14T08:48:10.361 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_19': 2024-09-14T08:48:10.361 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T08:48:10.361 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:48:10.361 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 1 2024-09-14T08:48:10.361 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 134aa96f3ba2 2024-09-14T08:48:10.361 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.134aa96f3ba2 2024-09-14T08:48:10.361 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:48:10.361 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering 2024-09-14T08:48:10.361 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:48:10.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:48:10.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:48:09 2024 2024-09-14T08:48:10.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:48:09 2024 2024-09-14T08:48:10.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:48:09 2024 2024-09-14T08:48:10.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:48:10.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:48:10.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: 72324075-a734-4269-8742-62ebe1d962b0 2024-09-14T08:48:10.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: true 2024-09-14T08:48:10.364 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n '' ']' 2024-09-14T08:48:10.365 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ stress_write_image cluster2 mirror image_19 2024-09-14T08:48:10.365 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:48:10.365 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:48:10.365 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_19 2024-09-14T08:48:10.365 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T08:48:10.366 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local duration=10 2024-09-14T08:48:10.366 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set +e 2024-09-14T08:48:10.367 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ timeout 10s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_19 --debug-rbd=20 --debug-journaler=20 2024-09-14T08:48:10.633 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:100 IOs, 65645568 bytes 2024-09-14T08:48:10.803 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:200 IOs, 107029504 bytes 2024-09-14T08:48:10.936 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:300 IOs, 138191872 bytes 2024-09-14T08:48:11.064 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:400 IOs, 151658496 bytes 2024-09-14T08:48:11.239 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:500 IOs, 195012608 bytes 2024-09-14T08:48:11.377 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:600 IOs, 241298432 bytes 2024-09-14T08:48:11.520 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:700 IOs, 255242240 bytes 2024-09-14T08:48:11.655 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:800 IOs, 289104896 bytes 2024-09-14T08:48:11.883 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:900 IOs, 340350976 bytes 2024-09-14T08:48:12.010 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1000 IOs, 365898752 bytes 2024-09-14T08:48:12.135 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1100 IOs, 387939328 bytes 2024-09-14T08:48:12.305 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1200 IOs, 426199040 bytes 2024-09-14T08:48:12.555 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1300 IOs, 511386624 bytes 2024-09-14T08:48:12.769 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1400 IOs, 559759360 bytes 2024-09-14T08:48:12.919 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1500 IOs, 590841856 bytes 2024-09-14T08:48:13.057 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1600 IOs, 615161856 bytes 2024-09-14T08:48:13.170 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1700 IOs, 633378816 bytes 2024-09-14T08:48:13.288 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1800 IOs, 673249280 bytes 2024-09-14T08:48:13.604 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1900 IOs, 755030016 bytes 2024-09-14T08:48:13.733 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2000 IOs, 779856896 bytes 2024-09-14T08:48:13.922 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2100 IOs, 829372416 bytes 2024-09-14T08:48:14.144 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2200 IOs, 899087360 bytes 2024-09-14T08:48:14.266 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2300 IOs, 918851584 bytes 2024-09-14T08:48:14.473 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2400 IOs, 973712384 bytes 2024-09-14T08:48:14.630 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2500 IOs, 1010367488 bytes 2024-09-14T08:48:14.820 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2600 IOs, 1076520960 bytes 2024-09-14T08:48:14.995 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2700 IOs, 1111205888 bytes 2024-09-14T08:48:15.209 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2800 IOs, 1158812672 bytes 2024-09-14T08:48:15.356 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2900 IOs, 1180178432 bytes 2024-09-14T08:48:15.457 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3000 IOs, 1199129600 bytes 2024-09-14T08:48:15.715 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3100 IOs, 1263821824 bytes 2024-09-14T08:48:16.022 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3200 IOs, 1367726080 bytes 2024-09-14T08:48:16.105 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3300 IOs, 1369491456 bytes 2024-09-14T08:48:16.133 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.7976771418980158) > weight (0.25) 2024-09-14T08:48:16.133 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.1 2024-09-14T08:48:16.133 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Sent signal 15 2024-09-14T08:48:16.133 INFO:tasks.rbd_mirror_thrash:waiting for 5 secs before reviving daemons 2024-09-14T08:48:16.135 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi164.stderr:2024-09-14T08:48:16.134+0000 7f4eb2fbf640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 (PID: 46309) UID: 1000 2024-09-14T08:48:16.286 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3400 IOs, 1409221632 bytes 2024-09-14T08:48:16.419 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3500 IOs, 1433261056 bytes 2024-09-14T08:48:16.636 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3600 IOs, 1494616064 bytes 2024-09-14T08:48:16.892 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3700 IOs, 1571767296 bytes 2024-09-14T08:48:16.959 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3800 IOs, 1572386816 bytes 2024-09-14T08:48:17.127 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3900 IOs, 1611332608 bytes 2024-09-14T08:48:17.227 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4000 IOs, 1623444480 bytes 2024-09-14T08:48:17.462 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4100 IOs, 1700956160 bytes 2024-09-14T08:48:17.560 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4200 IOs, 1703220224 bytes 2024-09-14T08:48:17.831 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4300 IOs, 1780343808 bytes 2024-09-14T08:48:18.005 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4400 IOs, 1819757568 bytes 2024-09-14T08:48:18.144 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4500 IOs, 1868310528 bytes 2024-09-14T08:48:18.406 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4600 IOs, 1928805376 bytes 2024-09-14T08:48:18.551 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4700 IOs, 1958026240 bytes 2024-09-14T08:48:18.666 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4800 IOs, 1973112832 bytes 2024-09-14T08:48:18.763 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4900 IOs, 1982445568 bytes 2024-09-14T08:48:18.936 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5000 IOs, 2018445312 bytes 2024-09-14T08:48:19.137 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5100 IOs, 2074116096 bytes 2024-09-14T08:48:19.458 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5200 IOs, 2169049088 bytes 2024-09-14T08:48:19.528 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5300 IOs, 2169674752 bytes 2024-09-14T08:48:19.721 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5400 IOs, 2219068416 bytes 2024-09-14T08:48:20.053 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5500 IOs, 2323733504 bytes 2024-09-14T08:48:20.128 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5600 IOs, 2324356096 bytes 2024-09-14T08:48:20.373 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ error_code=124 2024-09-14T08:48:20.373 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set -e 2024-09-14T08:48:20.373 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T08:48:20.373 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:48:20.373 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T08:48:20.373 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_20 2024-09-14T08:48:20.374 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image_and_enable_mirror cluster2 mirror image_20 snapshot 128M 2024-09-14T08:48:20.374 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:48:20.374 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:48:20.374 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:48:20.374 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:48:20.374 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_20 2024-09-14T08:48:20.374 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:48:20.374 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:48:20.374 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n snapshot ']' 2024-09-14T08:48:20.374 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:48:20.374 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image cluster2 mirror image_20 128M 2024-09-14T08:48:20.374 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:48:20.374 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:48:20.374 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:48:20.375 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:48:20.375 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_20 2024-09-14T08:48:20.375 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:48:20.375 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local size=128 2024-09-14T08:48:20.375 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n 128M ']' 2024-09-14T08:48:20.375 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ size=128M 2024-09-14T08:48:20.375 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:48:20.375 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_20 2024-09-14T08:48:20.426 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T08:48:20.426 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:48:20.426 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' image = image ']' 2024-09-14T08:48:20.426 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:48:20.426 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ enable_mirror cluster2 mirror image_20 snapshot 2024-09-14T08:48:20.426 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:48:20.427 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:48:20.427 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_20 2024-09-14T08:48:20.427 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:48:20.427 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_20 snapshot 2024-09-14T08:48:21.134 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.1 2024-09-14T08:48:21.134 INFO:teuthology.orchestra.run:waiting for 600 2024-09-14T08:48:21.134 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.1 2024-09-14T08:48:21.134 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Restarting daemon 2024-09-14T08:48:21.135 DEBUG:teuthology.orchestra.run.smithi164:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 2024-09-14T08:48:21.138 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Started 2024-09-14T08:48:21.138 INFO:tasks.rbd_mirror_thrash:waiting for 99 secs before thrashing 2024-09-14T08:48:21.334 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Mirroring enabled 2024-09-14T08:48:21.340 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 info mirror/image_20 2024-09-14T08:48:21.379 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_20': 2024-09-14T08:48:21.379 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T08:48:21.379 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:48:21.379 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 1 2024-09-14T08:48:21.380 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 13561ee069f1 2024-09-14T08:48:21.380 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.13561ee069f1 2024-09-14T08:48:21.380 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:48:21.380 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering 2024-09-14T08:48:21.380 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:48:21.380 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:48:21.380 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:48:20 2024 2024-09-14T08:48:21.380 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:48:20 2024 2024-09-14T08:48:21.380 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:48:20 2024 2024-09-14T08:48:21.380 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:48:21.380 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:48:21.380 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: f5c8b83b-151b-465c-b8b7-1ea8268e2846 2024-09-14T08:48:21.381 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: true 2024-09-14T08:48:21.383 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n '' ']' 2024-09-14T08:48:21.383 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ stress_write_image cluster2 mirror image_20 2024-09-14T08:48:21.383 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:48:21.383 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:48:21.383 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_20 2024-09-14T08:48:21.383 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T08:48:21.384 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local duration=9 2024-09-14T08:48:21.385 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set +e 2024-09-14T08:48:21.385 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ timeout 9s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_20 --debug-rbd=20 --debug-journaler=20 2024-09-14T08:48:21.569 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:100 IOs, 9708544 bytes 2024-09-14T08:48:21.709 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:200 IOs, 39499776 bytes 2024-09-14T08:48:21.846 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:300 IOs, 64424960 bytes 2024-09-14T08:48:21.976 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:400 IOs, 90248192 bytes 2024-09-14T08:48:22.098 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:500 IOs, 119496704 bytes 2024-09-14T08:48:22.349 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:600 IOs, 192605184 bytes 2024-09-14T08:48:22.533 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:700 IOs, 237519872 bytes 2024-09-14T08:48:22.702 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:800 IOs, 279370752 bytes 2024-09-14T08:48:22.827 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:900 IOs, 303345664 bytes 2024-09-14T08:48:22.969 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1000 IOs, 333700096 bytes 2024-09-14T08:48:23.069 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1100 IOs, 349686784 bytes 2024-09-14T08:48:23.240 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1200 IOs, 391898112 bytes 2024-09-14T08:48:23.420 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1300 IOs, 437898240 bytes 2024-09-14T08:48:23.539 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1400 IOs, 454624256 bytes 2024-09-14T08:48:23.681 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1500 IOs, 493967360 bytes 2024-09-14T08:48:23.923 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1600 IOs, 557008896 bytes 2024-09-14T08:48:24.088 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1700 IOs, 596697088 bytes 2024-09-14T08:48:24.425 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1800 IOs, 708147200 bytes 2024-09-14T08:48:24.657 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1900 IOs, 769468416 bytes 2024-09-14T08:48:24.782 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2000 IOs, 793616384 bytes 2024-09-14T08:48:24.902 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2100 IOs, 811765760 bytes 2024-09-14T08:48:25.006 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2200 IOs, 826502144 bytes 2024-09-14T08:48:25.247 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2300 IOs, 897057792 bytes 2024-09-14T08:48:25.473 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2400 IOs, 961132544 bytes 2024-09-14T08:48:25.791 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2500 IOs, 1068711936 bytes 2024-09-14T08:48:25.994 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2600 IOs, 1151429632 bytes 2024-09-14T08:48:26.297 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2700 IOs, 1210755072 bytes 2024-09-14T08:48:26.430 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2800 IOs, 1250327552 bytes 2024-09-14T08:48:26.565 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2900 IOs, 1260702720 bytes 2024-09-14T08:48:26.718 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3000 IOs, 1297062912 bytes 2024-09-14T08:48:26.864 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3100 IOs, 1335174144 bytes 2024-09-14T08:48:27.078 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3200 IOs, 1389831168 bytes 2024-09-14T08:48:27.164 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3300 IOs, 1413392384 bytes 2024-09-14T08:48:27.308 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3400 IOs, 1425530880 bytes 2024-09-14T08:48:27.461 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3500 IOs, 1460992000 bytes 2024-09-14T08:48:27.667 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3600 IOs, 1512896512 bytes 2024-09-14T08:48:27.822 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3700 IOs, 1558905856 bytes 2024-09-14T08:48:27.958 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3800 IOs, 1577554944 bytes 2024-09-14T08:48:28.159 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3900 IOs, 1629250560 bytes 2024-09-14T08:48:28.349 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4000 IOs, 1672920064 bytes 2024-09-14T08:48:28.524 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4100 IOs, 1712207872 bytes 2024-09-14T08:48:28.765 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4200 IOs, 1791188992 bytes 2024-09-14T08:48:28.916 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4300 IOs, 1856621568 bytes 2024-09-14T08:48:29.195 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4400 IOs, 1907767296 bytes 2024-09-14T08:48:29.361 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4500 IOs, 1945535488 bytes 2024-09-14T08:48:29.474 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4600 IOs, 1965322240 bytes 2024-09-14T08:48:29.665 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4700 IOs, 2013670400 bytes 2024-09-14T08:48:29.844 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4800 IOs, 2055227392 bytes 2024-09-14T08:48:30.171 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4900 IOs, 2161152000 bytes 2024-09-14T08:48:30.278 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5000 IOs, 2176617472 bytes 2024-09-14T08:48:30.390 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ error_code=124 2024-09-14T08:48:30.390 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set -e 2024-09-14T08:48:30.390 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T08:48:30.390 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:48:30.390 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T08:48:30.390 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_21 2024-09-14T08:48:30.391 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image_and_enable_mirror cluster2 mirror image_21 snapshot 128M 2024-09-14T08:48:30.391 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:48:30.391 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:48:30.391 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:48:30.391 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:48:30.391 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_21 2024-09-14T08:48:30.391 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:48:30.391 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:48:30.391 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n snapshot ']' 2024-09-14T08:48:30.391 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:48:30.391 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image cluster2 mirror image_21 128M 2024-09-14T08:48:30.391 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:48:30.391 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:48:30.392 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:48:30.392 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:48:30.392 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_21 2024-09-14T08:48:30.392 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:48:30.392 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local size=128 2024-09-14T08:48:30.392 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n 128M ']' 2024-09-14T08:48:30.392 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ size=128M 2024-09-14T08:48:30.392 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:48:30.392 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_21 2024-09-14T08:48:30.443 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T08:48:30.443 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:48:30.443 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' image = image ']' 2024-09-14T08:48:30.443 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:48:30.443 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ enable_mirror cluster2 mirror image_21 snapshot 2024-09-14T08:48:30.443 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:48:30.444 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:48:30.444 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_21 2024-09-14T08:48:30.444 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:48:30.444 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_21 snapshot 2024-09-14T08:48:31.348 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Mirroring enabled 2024-09-14T08:48:31.354 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 info mirror/image_21 2024-09-14T08:48:31.396 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_21': 2024-09-14T08:48:31.396 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T08:48:31.396 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:48:31.396 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 1 2024-09-14T08:48:31.396 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 13676f308d55 2024-09-14T08:48:31.397 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.13676f308d55 2024-09-14T08:48:31.397 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:48:31.397 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering 2024-09-14T08:48:31.397 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:48:31.397 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:48:31.397 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:48:30 2024 2024-09-14T08:48:31.397 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:48:30 2024 2024-09-14T08:48:31.397 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:48:30 2024 2024-09-14T08:48:31.397 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:48:31.397 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:48:31.397 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: 75208eea-1f60-4048-a017-3885fa315408 2024-09-14T08:48:31.398 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: true 2024-09-14T08:48:31.400 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n '' ']' 2024-09-14T08:48:31.400 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ stress_write_image cluster2 mirror image_21 2024-09-14T08:48:31.400 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:48:31.400 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:48:31.400 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_21 2024-09-14T08:48:31.400 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T08:48:31.402 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local duration=5 2024-09-14T08:48:31.402 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set +e 2024-09-14T08:48:31.402 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ timeout 5s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_21 --debug-rbd=20 --debug-journaler=20 2024-09-14T08:48:31.613 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:100 IOs, 41249792 bytes 2024-09-14T08:48:31.733 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:200 IOs, 61645824 bytes 2024-09-14T08:48:31.981 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:300 IOs, 135791616 bytes 2024-09-14T08:48:32.157 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:400 IOs, 178287616 bytes 2024-09-14T08:48:32.334 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:500 IOs, 217586688 bytes 2024-09-14T08:48:32.465 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:600 IOs, 266140672 bytes 2024-09-14T08:48:32.599 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:700 IOs, 266759168 bytes 2024-09-14T08:48:32.971 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:800 IOs, 391249920 bytes 2024-09-14T08:48:33.089 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:900 IOs, 408232960 bytes 2024-09-14T08:48:33.160 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1000 IOs, 408848384 bytes 2024-09-14T08:48:33.325 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1100 IOs, 447961088 bytes 2024-09-14T08:48:33.518 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1200 IOs, 506184704 bytes 2024-09-14T08:48:33.647 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1300 IOs, 518464512 bytes 2024-09-14T08:48:33.769 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1400 IOs, 546947072 bytes 2024-09-14T08:48:33.988 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1500 IOs, 606462976 bytes 2024-09-14T08:48:34.135 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1600 IOs, 629331968 bytes 2024-09-14T08:48:34.371 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1700 IOs, 705755136 bytes 2024-09-14T08:48:34.565 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1800 IOs, 741945344 bytes 2024-09-14T08:48:34.796 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1900 IOs, 805647360 bytes 2024-09-14T08:48:35.095 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2000 IOs, 912333824 bytes 2024-09-14T08:48:35.264 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2100 IOs, 935895040 bytes 2024-09-14T08:48:35.534 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2200 IOs, 1011762176 bytes 2024-09-14T08:48:35.681 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2300 IOs, 1043385344 bytes 2024-09-14T08:48:35.844 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2400 IOs, 1069300736 bytes 2024-09-14T08:48:36.170 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2500 IOs, 1189541888 bytes 2024-09-14T08:48:36.407 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ error_code=124 2024-09-14T08:48:36.408 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set -e 2024-09-14T08:48:36.408 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T08:48:36.408 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:48:36.408 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T08:48:36.408 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_22 2024-09-14T08:48:36.408 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image_and_enable_mirror cluster2 mirror image_22 snapshot 128M 2024-09-14T08:48:36.408 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:48:36.408 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:48:36.408 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:48:36.408 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:48:36.408 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_22 2024-09-14T08:48:36.408 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:48:36.409 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:48:36.409 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n snapshot ']' 2024-09-14T08:48:36.409 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:48:36.409 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image cluster2 mirror image_22 128M 2024-09-14T08:48:36.409 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:48:36.409 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:48:36.409 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:48:36.409 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:48:36.409 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_22 2024-09-14T08:48:36.409 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:48:36.409 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local size=128 2024-09-14T08:48:36.409 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n 128M ']' 2024-09-14T08:48:36.409 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ size=128M 2024-09-14T08:48:36.409 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:48:36.409 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_22 2024-09-14T08:48:36.460 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T08:48:36.460 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:48:36.460 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' image = image ']' 2024-09-14T08:48:36.460 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:48:36.460 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ enable_mirror cluster2 mirror image_22 snapshot 2024-09-14T08:48:36.460 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:48:36.460 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:48:36.460 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_22 2024-09-14T08:48:36.460 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:48:36.460 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_22 snapshot 2024-09-14T08:48:37.356 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Mirroring enabled 2024-09-14T08:48:37.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 info mirror/image_22 2024-09-14T08:48:37.403 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_22': 2024-09-14T08:48:37.403 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T08:48:37.403 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:48:37.403 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 1 2024-09-14T08:48:37.403 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 13723efdf754 2024-09-14T08:48:37.403 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.13723efdf754 2024-09-14T08:48:37.403 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:48:37.404 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering 2024-09-14T08:48:37.404 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:48:37.404 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:48:37.404 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:48:36 2024 2024-09-14T08:48:37.404 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:48:36 2024 2024-09-14T08:48:37.404 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:48:36 2024 2024-09-14T08:48:37.404 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:48:37.404 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:48:37.404 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: 83c2e42b-5fd9-42ba-b3f2-2e9c690a99bf 2024-09-14T08:48:37.404 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: true 2024-09-14T08:48:37.407 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n '' ']' 2024-09-14T08:48:37.407 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ stress_write_image cluster2 mirror image_22 2024-09-14T08:48:37.407 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:48:37.407 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:48:37.407 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_22 2024-09-14T08:48:37.407 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T08:48:37.409 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local duration=5 2024-09-14T08:48:37.409 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set +e 2024-09-14T08:48:37.409 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ timeout 5s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_22 --debug-rbd=20 --debug-journaler=20 2024-09-14T08:48:37.585 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:100 IOs, 22088704 bytes 2024-09-14T08:48:37.787 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:200 IOs, 75987968 bytes 2024-09-14T08:48:37.935 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:300 IOs, 98940928 bytes 2024-09-14T08:48:38.025 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:400 IOs, 106353664 bytes 2024-09-14T08:48:38.146 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:500 IOs, 121468928 bytes 2024-09-14T08:48:38.249 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:600 IOs, 129191936 bytes 2024-09-14T08:48:38.558 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:700 IOs, 197508096 bytes 2024-09-14T08:48:38.696 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:800 IOs, 226971648 bytes 2024-09-14T08:48:39.142 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:900 IOs, 361280512 bytes 2024-09-14T08:48:39.434 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1000 IOs, 434024448 bytes 2024-09-14T08:48:39.661 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1100 IOs, 493486080 bytes 2024-09-14T08:48:39.795 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1200 IOs, 511405056 bytes 2024-09-14T08:48:39.938 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1300 IOs, 541550592 bytes 2024-09-14T08:48:40.196 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1400 IOs, 605358080 bytes 2024-09-14T08:48:40.444 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1500 IOs, 671299584 bytes 2024-09-14T08:48:40.573 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1600 IOs, 692130816 bytes 2024-09-14T08:48:40.877 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1700 IOs, 787123200 bytes 2024-09-14T08:48:40.916 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.738987671982316) > weight (0.25) 2024-09-14T08:48:40.917 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.4425470649311044) > weight (0.25) 2024-09-14T08:48:40.917 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.6657752624258123) > weight (0.25) 2024-09-14T08:48:40.917 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.3 with skip (0.29036903015652815) > weight (0.25) 2024-09-14T08:48:40.917 INFO:tasks.rbd_mirror_thrash:waiting for 105 secs before thrashing 2024-09-14T08:48:41.131 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1800 IOs, 843514880 bytes 2024-09-14T08:48:41.324 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1900 IOs, 910380032 bytes 2024-09-14T08:48:41.818 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2000 IOs, 972209152 bytes 2024-09-14T08:48:41.921 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2100 IOs, 988290048 bytes 2024-09-14T08:48:42.051 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2200 IOs, 1010087936 bytes 2024-09-14T08:48:42.176 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2300 IOs, 1026067456 bytes 2024-09-14T08:48:42.305 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2400 IOs, 1045467136 bytes 2024-09-14T08:48:42.415 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ error_code=124 2024-09-14T08:48:42.415 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set -e 2024-09-14T08:48:42.415 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T08:48:42.415 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:48:42.415 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T08:48:42.415 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_23 2024-09-14T08:48:42.416 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image_and_enable_mirror cluster2 mirror image_23 snapshot 128M 2024-09-14T08:48:42.416 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:48:42.416 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:48:42.416 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:48:42.416 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:48:42.416 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_23 2024-09-14T08:48:42.416 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:48:42.416 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:48:42.416 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n snapshot ']' 2024-09-14T08:48:42.416 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:48:42.416 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image cluster2 mirror image_23 128M 2024-09-14T08:48:42.416 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:48:42.416 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:48:42.416 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:48:42.416 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:48:42.417 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_23 2024-09-14T08:48:42.417 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:48:42.417 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local size=128 2024-09-14T08:48:42.417 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n 128M ']' 2024-09-14T08:48:42.417 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ size=128M 2024-09-14T08:48:42.417 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:48:42.417 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_23 2024-09-14T08:48:42.463 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T08:48:42.463 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:48:42.463 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' image = image ']' 2024-09-14T08:48:42.463 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:48:42.464 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ enable_mirror cluster2 mirror image_23 snapshot 2024-09-14T08:48:42.464 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:48:42.464 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:48:42.464 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_23 2024-09-14T08:48:42.464 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:48:42.464 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_23 snapshot 2024-09-14T08:48:43.367 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Mirroring enabled 2024-09-14T08:48:43.373 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 info mirror/image_23 2024-09-14T08:48:43.412 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_23': 2024-09-14T08:48:43.412 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T08:48:43.413 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:48:43.413 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 1 2024-09-14T08:48:43.413 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 137e8faa86a6 2024-09-14T08:48:43.413 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.137e8faa86a6 2024-09-14T08:48:43.413 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:48:43.413 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering 2024-09-14T08:48:43.413 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:48:43.413 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:48:43.413 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:48:42 2024 2024-09-14T08:48:43.413 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:48:42 2024 2024-09-14T08:48:43.413 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:48:42 2024 2024-09-14T08:48:43.413 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:48:43.413 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:48:43.413 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: b97e8af3-4a7b-444b-9f49-8151fd4bf536 2024-09-14T08:48:43.413 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: true 2024-09-14T08:48:43.416 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n '' ']' 2024-09-14T08:48:43.416 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ stress_write_image cluster2 mirror image_23 2024-09-14T08:48:43.416 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:48:43.416 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:48:43.416 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_23 2024-09-14T08:48:43.416 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T08:48:43.418 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local duration=11 2024-09-14T08:48:43.418 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set +e 2024-09-14T08:48:43.418 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ timeout 11s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_23 --debug-rbd=20 --debug-journaler=20 2024-09-14T08:48:43.629 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:100 IOs, 43344896 bytes 2024-09-14T08:48:43.786 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:200 IOs, 85990400 bytes 2024-09-14T08:48:44.097 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:300 IOs, 183805952 bytes 2024-09-14T08:48:44.240 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:400 IOs, 216614912 bytes 2024-09-14T08:48:44.472 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:500 IOs, 279598080 bytes 2024-09-14T08:48:44.754 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:600 IOs, 370148352 bytes 2024-09-14T08:48:44.964 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:700 IOs, 452411392 bytes 2024-09-14T08:48:45.115 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:800 IOs, 464797696 bytes 2024-09-14T08:48:45.261 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:900 IOs, 518750208 bytes 2024-09-14T08:48:45.419 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1000 IOs, 532214784 bytes 2024-09-14T08:48:45.609 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1100 IOs, 586893312 bytes 2024-09-14T08:48:45.758 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1200 IOs, 620319744 bytes 2024-09-14T08:48:45.765 INFO:tasks.ceph.cluster2.osd.0.smithi164.stderr:2024-09-14T08:48:45.764+0000 7f7ce7714640 -1 reset not still connected to 0x55a089161450 2024-09-14T08:48:45.828 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1300 IOs, 620938240 bytes 2024-09-14T08:48:45.963 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1400 IOs, 647247872 bytes 2024-09-14T08:48:46.149 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1500 IOs, 717833216 bytes 2024-09-14T08:48:46.304 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1600 IOs, 751109120 bytes 2024-09-14T08:48:46.511 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1700 IOs, 786416640 bytes 2024-09-14T08:48:46.582 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1800 IOs, 787031040 bytes 2024-09-14T08:48:46.680 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1900 IOs, 808546304 bytes 2024-09-14T08:48:46.913 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2000 IOs, 867409920 bytes 2024-09-14T08:48:47.058 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2100 IOs, 898872320 bytes 2024-09-14T08:48:47.130 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2200 IOs, 924161024 bytes 2024-09-14T08:48:47.323 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2300 IOs, 965390336 bytes 2024-09-14T08:48:47.541 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2400 IOs, 1007357952 bytes 2024-09-14T08:48:47.682 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2500 IOs, 1037128704 bytes 2024-09-14T08:48:47.937 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2600 IOs, 1120740352 bytes 2024-09-14T08:48:48.035 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2700 IOs, 1147811840 bytes 2024-09-14T08:48:48.211 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2800 IOs, 1171559424 bytes 2024-09-14T08:48:48.436 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2900 IOs, 1235927040 bytes 2024-09-14T08:48:48.616 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3000 IOs, 1296482304 bytes 2024-09-14T08:48:48.787 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3100 IOs, 1320755200 bytes 2024-09-14T08:48:48.887 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3200 IOs, 1333221376 bytes 2024-09-14T08:48:48.964 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3300 IOs, 1346089984 bytes 2024-09-14T08:48:49.173 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3400 IOs, 1397478400 bytes 2024-09-14T08:48:49.371 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3500 IOs, 1451599872 bytes 2024-09-14T08:48:49.546 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3600 IOs, 1487753216 bytes 2024-09-14T08:48:49.771 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3700 IOs, 1549604864 bytes 2024-09-14T08:48:49.865 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3800 IOs, 1571369984 bytes 2024-09-14T08:48:50.002 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3900 IOs, 1586919424 bytes 2024-09-14T08:48:50.074 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4000 IOs, 1587536896 bytes 2024-09-14T08:48:50.405 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4100 IOs, 1688078336 bytes 2024-09-14T08:48:50.627 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4200 IOs, 1758547968 bytes 2024-09-14T08:48:50.841 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4300 IOs, 1808821248 bytes 2024-09-14T08:48:50.995 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4400 IOs, 1848656896 bytes 2024-09-14T08:48:51.137 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4500 IOs, 1872420864 bytes 2024-09-14T08:48:51.372 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4600 IOs, 1935605760 bytes 2024-09-14T08:48:51.510 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4700 IOs, 1966994432 bytes 2024-09-14T08:48:51.646 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4800 IOs, 1993026560 bytes 2024-09-14T08:48:51.760 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4900 IOs, 2026102784 bytes 2024-09-14T08:48:51.963 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5000 IOs, 2065097728 bytes 2024-09-14T08:48:52.109 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5100 IOs, 2097129472 bytes 2024-09-14T08:48:52.332 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5200 IOs, 2160643072 bytes 2024-09-14T08:48:52.430 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5300 IOs, 2172346368 bytes 2024-09-14T08:48:52.654 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5400 IOs, 2237110272 bytes 2024-09-14T08:48:52.772 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5500 IOs, 2255738880 bytes 2024-09-14T08:48:52.971 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5600 IOs, 2312059904 bytes 2024-09-14T08:48:53.044 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5700 IOs, 2312673280 bytes 2024-09-14T08:48:53.252 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5800 IOs, 2370506752 bytes 2024-09-14T08:48:53.333 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5900 IOs, 2374240256 bytes 2024-09-14T08:48:53.582 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6000 IOs, 2445587456 bytes 2024-09-14T08:48:53.775 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6100 IOs, 2494135296 bytes 2024-09-14T08:48:54.117 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6200 IOs, 2524973056 bytes 2024-09-14T08:48:54.221 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6300 IOs, 2539035648 bytes 2024-09-14T08:48:54.327 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6400 IOs, 2555732992 bytes 2024-09-14T08:48:54.425 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ error_code=124 2024-09-14T08:48:54.425 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set -e 2024-09-14T08:48:54.425 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T08:48:54.425 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:48:54.425 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T08:48:54.425 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_24 2024-09-14T08:48:54.425 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image_and_enable_mirror cluster2 mirror image_24 snapshot 128M 2024-09-14T08:48:54.425 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:48:54.425 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:48:54.426 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:48:54.426 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:48:54.426 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_24 2024-09-14T08:48:54.426 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:48:54.426 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:48:54.426 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n snapshot ']' 2024-09-14T08:48:54.426 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:48:54.426 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image cluster2 mirror image_24 128M 2024-09-14T08:48:54.426 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:48:54.426 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:48:54.426 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:48:54.426 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:48:54.426 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_24 2024-09-14T08:48:54.427 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:48:54.427 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local size=128 2024-09-14T08:48:54.427 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n 128M ']' 2024-09-14T08:48:54.427 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ size=128M 2024-09-14T08:48:54.427 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:48:54.427 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_24 2024-09-14T08:48:54.475 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T08:48:54.475 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:48:54.475 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' image = image ']' 2024-09-14T08:48:54.475 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:48:54.475 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ enable_mirror cluster2 mirror image_24 snapshot 2024-09-14T08:48:54.475 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:48:54.475 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:48:54.475 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_24 2024-09-14T08:48:54.475 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:48:54.475 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_24 snapshot 2024-09-14T08:48:55.383 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Mirroring enabled 2024-09-14T08:48:55.390 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 info mirror/image_24 2024-09-14T08:48:55.433 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_24': 2024-09-14T08:48:55.433 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T08:48:55.433 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:48:55.433 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 1 2024-09-14T08:48:55.434 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 138ab8c91acb 2024-09-14T08:48:55.434 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.138ab8c91acb 2024-09-14T08:48:55.434 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:48:55.434 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering 2024-09-14T08:48:55.434 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:48:55.434 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:48:55.434 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:48:54 2024 2024-09-14T08:48:55.434 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:48:54 2024 2024-09-14T08:48:55.434 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:48:54 2024 2024-09-14T08:48:55.435 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:48:55.435 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:48:55.435 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: 24c48f34-7f6e-4594-86fc-0a765052746d 2024-09-14T08:48:55.435 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: true 2024-09-14T08:48:55.437 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n '' ']' 2024-09-14T08:48:55.437 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ stress_write_image cluster2 mirror image_24 2024-09-14T08:48:55.437 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:48:55.438 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:48:55.438 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_24 2024-09-14T08:48:55.438 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T08:48:55.439 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local duration=9 2024-09-14T08:48:55.439 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set +e 2024-09-14T08:48:55.439 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ timeout 9s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_24 --debug-rbd=20 --debug-journaler=20 2024-09-14T08:48:55.633 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:100 IOs, 28326912 bytes 2024-09-14T08:48:55.838 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:200 IOs, 87009280 bytes 2024-09-14T08:48:55.990 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:300 IOs, 118803456 bytes 2024-09-14T08:48:56.071 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:400 IOs, 122667008 bytes 2024-09-14T08:48:56.156 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:500 IOs, 128257024 bytes 2024-09-14T08:48:56.285 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:600 IOs, 164380672 bytes 2024-09-14T08:48:56.531 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:700 IOs, 220665856 bytes 2024-09-14T08:48:56.778 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:800 IOs, 283858944 bytes 2024-09-14T08:48:56.882 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:900 IOs, 293980160 bytes 2024-09-14T08:48:57.114 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1000 IOs, 338096128 bytes 2024-09-14T08:48:57.266 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1100 IOs, 389551104 bytes 2024-09-14T08:48:57.529 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1200 IOs, 443257856 bytes 2024-09-14T08:48:57.770 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1300 IOs, 511124480 bytes 2024-09-14T08:48:57.899 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1400 IOs, 558029824 bytes 2024-09-14T08:48:58.048 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1500 IOs, 562842624 bytes 2024-09-14T08:48:58.127 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1600 IOs, 568664064 bytes 2024-09-14T08:48:58.264 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1700 IOs, 591921152 bytes 2024-09-14T08:48:58.345 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1800 IOs, 594820096 bytes 2024-09-14T08:48:58.586 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1900 IOs, 662243328 bytes 2024-09-14T08:48:58.862 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2000 IOs, 741801984 bytes 2024-09-14T08:48:59.146 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2100 IOs, 829787136 bytes 2024-09-14T08:48:59.350 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2200 IOs, 886286336 bytes 2024-09-14T08:48:59.524 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2300 IOs, 926891008 bytes 2024-09-14T08:48:59.804 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2400 IOs, 1009790976 bytes 2024-09-14T08:48:59.932 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2500 IOs, 1032446976 bytes 2024-09-14T08:49:00.146 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2600 IOs, 1090495488 bytes 2024-09-14T08:49:00.280 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2700 IOs, 1124853760 bytes 2024-09-14T08:49:00.546 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2800 IOs, 1207686144 bytes 2024-09-14T08:49:00.807 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2900 IOs, 1289038848 bytes 2024-09-14T08:49:01.013 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3000 IOs, 1321483264 bytes 2024-09-14T08:49:01.137 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3100 IOs, 1384071168 bytes 2024-09-14T08:49:01.385 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3200 IOs, 1413406720 bytes 2024-09-14T08:49:01.530 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3300 IOs, 1440924672 bytes 2024-09-14T08:49:01.722 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3400 IOs, 1485759488 bytes 2024-09-14T08:49:01.906 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3500 IOs, 1530014720 bytes 2024-09-14T08:49:02.101 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3600 IOs, 1576506368 bytes 2024-09-14T08:49:02.283 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3700 IOs, 1620549632 bytes 2024-09-14T08:49:02.513 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3800 IOs, 1687127040 bytes 2024-09-14T08:49:02.657 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3900 IOs, 1749911552 bytes 2024-09-14T08:49:02.910 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4000 IOs, 1799043072 bytes 2024-09-14T08:49:03.151 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4100 IOs, 1857551360 bytes 2024-09-14T08:49:03.307 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4200 IOs, 1902797824 bytes 2024-09-14T08:49:03.473 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4300 IOs, 1936090112 bytes 2024-09-14T08:49:03.672 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4400 IOs, 1985388544 bytes 2024-09-14T08:49:03.812 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4500 IOs, 2012703744 bytes 2024-09-14T08:49:03.984 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4600 IOs, 2057209856 bytes 2024-09-14T08:49:04.181 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4700 IOs, 2105771008 bytes 2024-09-14T08:49:04.414 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4800 IOs, 2170900480 bytes 2024-09-14T08:49:04.446 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ error_code=124 2024-09-14T08:49:04.446 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set -e 2024-09-14T08:49:04.446 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T08:49:04.446 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:49:04.446 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T08:49:04.446 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_25 2024-09-14T08:49:04.446 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image_and_enable_mirror cluster2 mirror image_25 snapshot 128M 2024-09-14T08:49:04.446 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:49:04.446 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:49:04.446 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:49:04.447 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:49:04.447 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_25 2024-09-14T08:49:04.447 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:49:04.447 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:49:04.447 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n snapshot ']' 2024-09-14T08:49:04.447 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:49:04.447 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image cluster2 mirror image_25 128M 2024-09-14T08:49:04.447 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:49:04.447 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:49:04.447 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:49:04.447 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:49:04.447 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_25 2024-09-14T08:49:04.447 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:49:04.447 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local size=128 2024-09-14T08:49:04.447 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n 128M ']' 2024-09-14T08:49:04.448 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ size=128M 2024-09-14T08:49:04.448 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:49:04.448 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_25 2024-09-14T08:49:04.504 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T08:49:04.504 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:49:04.504 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' image = image ']' 2024-09-14T08:49:04.504 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:49:04.504 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ enable_mirror cluster2 mirror image_25 snapshot 2024-09-14T08:49:04.504 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:49:04.504 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:49:04.504 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_25 2024-09-14T08:49:04.504 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:49:04.504 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_25 snapshot 2024-09-14T08:49:05.393 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Mirroring enabled 2024-09-14T08:49:05.399 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 info mirror/image_25 2024-09-14T08:49:05.440 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_25': 2024-09-14T08:49:05.440 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T08:49:05.440 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:49:05.440 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 1 2024-09-14T08:49:05.440 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 1396dd677e7e 2024-09-14T08:49:05.440 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.1396dd677e7e 2024-09-14T08:49:05.440 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:49:05.440 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering 2024-09-14T08:49:05.440 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:49:05.440 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:49:05.440 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:49:04 2024 2024-09-14T08:49:05.441 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:49:04 2024 2024-09-14T08:49:05.441 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:49:04 2024 2024-09-14T08:49:05.441 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:49:05.441 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:49:05.441 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: 3d365e96-a01e-4587-97b2-7c3efff31e05 2024-09-14T08:49:05.441 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: true 2024-09-14T08:49:05.443 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n '' ']' 2024-09-14T08:49:05.443 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ stress_write_image cluster2 mirror image_25 2024-09-14T08:49:05.443 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:49:05.443 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:49:05.443 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_25 2024-09-14T08:49:05.444 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T08:49:05.445 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local duration=6 2024-09-14T08:49:05.445 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set +e 2024-09-14T08:49:05.445 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ timeout 6s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_25 --debug-rbd=20 --debug-journaler=20 2024-09-14T08:49:05.819 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:100 IOs, 32350208 bytes 2024-09-14T08:49:06.043 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:200 IOs, 85545984 bytes 2024-09-14T08:49:06.289 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:300 IOs, 156265472 bytes 2024-09-14T08:49:06.587 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:400 IOs, 258934784 bytes 2024-09-14T08:49:06.822 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:500 IOs, 321007616 bytes 2024-09-14T08:49:07.041 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:600 IOs, 379440128 bytes 2024-09-14T08:49:07.283 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:700 IOs, 439191552 bytes 2024-09-14T08:49:07.418 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:800 IOs, 464718848 bytes 2024-09-14T08:49:07.536 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:900 IOs, 483844096 bytes 2024-09-14T08:49:07.774 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1000 IOs, 563746816 bytes 2024-09-14T08:49:07.877 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1100 IOs, 575010816 bytes 2024-09-14T08:49:08.100 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1200 IOs, 642826240 bytes 2024-09-14T08:49:08.288 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1300 IOs, 680658944 bytes 2024-09-14T08:49:08.573 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1400 IOs, 760559616 bytes 2024-09-14T08:49:08.733 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1500 IOs, 797064192 bytes 2024-09-14T08:49:08.834 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1600 IOs, 810012672 bytes 2024-09-14T08:49:08.910 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1700 IOs, 821069824 bytes 2024-09-14T08:49:09.194 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1800 IOs, 898463744 bytes 2024-09-14T08:49:09.339 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1900 IOs, 925682688 bytes 2024-09-14T08:49:09.546 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2000 IOs, 978139136 bytes 2024-09-14T08:49:09.696 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2100 IOs, 1011769344 bytes 2024-09-14T08:49:09.951 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2200 IOs, 1082494976 bytes 2024-09-14T08:49:10.046 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2300 IOs, 1089863680 bytes 2024-09-14T08:49:10.160 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2400 IOs, 1106120704 bytes 2024-09-14T08:49:10.301 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2500 IOs, 1137079296 bytes 2024-09-14T08:49:10.557 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2600 IOs, 1215397888 bytes 2024-09-14T08:49:10.647 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2700 IOs, 1219537920 bytes 2024-09-14T08:49:10.936 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2800 IOs, 1311199232 bytes 2024-09-14T08:49:11.077 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2900 IOs, 1342953472 bytes 2024-09-14T08:49:11.253 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3000 IOs, 1378717696 bytes 2024-09-14T08:49:11.452 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ error_code=124 2024-09-14T08:49:11.452 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set -e 2024-09-14T08:49:11.452 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T08:49:11.452 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:49:11.452 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T08:49:11.453 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_26 2024-09-14T08:49:11.453 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image_and_enable_mirror cluster2 mirror image_26 snapshot 128M 2024-09-14T08:49:11.453 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:49:11.453 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:49:11.453 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:49:11.453 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:49:11.453 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_26 2024-09-14T08:49:11.453 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:49:11.453 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:49:11.453 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n snapshot ']' 2024-09-14T08:49:11.453 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:49:11.453 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image cluster2 mirror image_26 128M 2024-09-14T08:49:11.453 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:49:11.453 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:49:11.454 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:49:11.454 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:49:11.454 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_26 2024-09-14T08:49:11.454 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:49:11.454 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local size=128 2024-09-14T08:49:11.454 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n 128M ']' 2024-09-14T08:49:11.454 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ size=128M 2024-09-14T08:49:11.455 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:49:11.455 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_26 2024-09-14T08:49:11.502 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T08:49:11.503 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:49:11.503 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' image = image ']' 2024-09-14T08:49:11.503 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:49:11.503 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ enable_mirror cluster2 mirror image_26 snapshot 2024-09-14T08:49:11.503 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:49:11.503 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:49:11.503 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_26 2024-09-14T08:49:11.503 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:49:11.503 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_26 snapshot 2024-09-14T08:49:12.400 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Mirroring enabled 2024-09-14T08:49:12.407 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 info mirror/image_26 2024-09-14T08:49:12.449 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_26': 2024-09-14T08:49:12.449 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T08:49:12.449 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:49:12.449 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 1 2024-09-14T08:49:12.450 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 13a3297ac120 2024-09-14T08:49:12.450 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.13a3297ac120 2024-09-14T08:49:12.450 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:49:12.450 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering 2024-09-14T08:49:12.450 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:49:12.450 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:49:12.450 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:49:11 2024 2024-09-14T08:49:12.450 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:49:11 2024 2024-09-14T08:49:12.450 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:49:11 2024 2024-09-14T08:49:12.450 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:49:12.450 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:49:12.450 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: 4ec19e01-86f0-46e8-9aed-ae17d2da69c7 2024-09-14T08:49:12.450 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: true 2024-09-14T08:49:12.453 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n '' ']' 2024-09-14T08:49:12.453 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ stress_write_image cluster2 mirror image_26 2024-09-14T08:49:12.453 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:49:12.453 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:49:12.453 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_26 2024-09-14T08:49:12.453 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T08:49:12.455 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local duration=6 2024-09-14T08:49:12.455 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set +e 2024-09-14T08:49:12.455 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ timeout 6s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_26 --debug-rbd=20 --debug-journaler=20 2024-09-14T08:49:12.743 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:100 IOs, 98234368 bytes 2024-09-14T08:49:13.077 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:200 IOs, 179072000 bytes 2024-09-14T08:49:13.153 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:300 IOs, 179689472 bytes 2024-09-14T08:49:13.223 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:400 IOs, 180302848 bytes 2024-09-14T08:49:13.340 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:500 IOs, 197332992 bytes 2024-09-14T08:49:13.492 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:600 IOs, 228240384 bytes 2024-09-14T08:49:13.704 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:700 IOs, 280852480 bytes 2024-09-14T08:49:13.775 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:800 IOs, 286350336 bytes 2024-09-14T08:49:14.044 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:900 IOs, 357505024 bytes 2024-09-14T08:49:14.246 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1000 IOs, 406572032 bytes 2024-09-14T08:49:14.394 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1100 IOs, 438370304 bytes 2024-09-14T08:49:14.499 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1200 IOs, 452535296 bytes 2024-09-14T08:49:14.699 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1300 IOs, 509119488 bytes 2024-09-14T08:49:14.846 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1400 IOs, 531695616 bytes 2024-09-14T08:49:15.099 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1500 IOs, 606906368 bytes 2024-09-14T08:49:15.328 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1600 IOs, 655673344 bytes 2024-09-14T08:49:15.543 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1700 IOs, 720229376 bytes 2024-09-14T08:49:15.690 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1800 IOs, 745098240 bytes 2024-09-14T08:49:16.044 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1900 IOs, 862143488 bytes 2024-09-14T08:49:16.235 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2000 IOs, 904647680 bytes 2024-09-14T08:49:16.457 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2100 IOs, 971972608 bytes 2024-09-14T08:49:16.586 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2200 IOs, 994279424 bytes 2024-09-14T08:49:16.776 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2300 IOs, 1031344128 bytes 2024-09-14T08:49:16.972 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2400 IOs, 1084233728 bytes 2024-09-14T08:49:17.134 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2500 IOs, 1118451712 bytes 2024-09-14T08:49:17.277 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2600 IOs, 1156002816 bytes 2024-09-14T08:49:17.482 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2700 IOs, 1201514496 bytes 2024-09-14T08:49:17.664 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2800 IOs, 1247270912 bytes 2024-09-14T08:49:17.770 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2900 IOs, 1260764160 bytes 2024-09-14T08:49:18.013 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3000 IOs, 1340080128 bytes 2024-09-14T08:49:18.226 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3100 IOs, 1391468544 bytes 2024-09-14T08:49:18.461 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ error_code=124 2024-09-14T08:49:18.461 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set -e 2024-09-14T08:49:18.461 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T08:49:18.461 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:49:18.461 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T08:49:18.461 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_27 2024-09-14T08:49:18.461 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image_and_enable_mirror cluster2 mirror image_27 snapshot 128M 2024-09-14T08:49:18.461 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:49:18.461 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:49:18.461 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:49:18.461 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:49:18.461 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_27 2024-09-14T08:49:18.462 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:49:18.462 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:49:18.462 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n snapshot ']' 2024-09-14T08:49:18.462 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:49:18.462 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image cluster2 mirror image_27 128M 2024-09-14T08:49:18.462 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:49:18.462 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:49:18.462 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:49:18.462 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:49:18.462 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_27 2024-09-14T08:49:18.462 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:49:18.462 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local size=128 2024-09-14T08:49:18.462 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n 128M ']' 2024-09-14T08:49:18.462 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ size=128M 2024-09-14T08:49:18.463 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:49:18.463 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_27 2024-09-14T08:49:18.510 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T08:49:18.510 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:49:18.510 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' image = image ']' 2024-09-14T08:49:18.510 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:49:18.510 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ enable_mirror cluster2 mirror image_27 snapshot 2024-09-14T08:49:18.510 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:49:18.511 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:49:18.511 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_27 2024-09-14T08:49:18.511 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:49:18.511 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_27 snapshot 2024-09-14T08:49:19.519 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Mirroring enabled 2024-09-14T08:49:19.524 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 info mirror/image_27 2024-09-14T08:49:19.563 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_27': 2024-09-14T08:49:19.563 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T08:49:19.563 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:49:19.563 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 1 2024-09-14T08:49:19.563 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 13af4f6b2f8a 2024-09-14T08:49:19.563 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.13af4f6b2f8a 2024-09-14T08:49:19.563 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:49:19.563 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering 2024-09-14T08:49:19.563 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:49:19.564 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:49:19.564 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:49:18 2024 2024-09-14T08:49:19.564 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:49:18 2024 2024-09-14T08:49:19.564 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:49:18 2024 2024-09-14T08:49:19.564 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:49:19.564 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:49:19.564 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: a44041c2-b983-4eef-b3f2-aec32b4ab580 2024-09-14T08:49:19.564 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: true 2024-09-14T08:49:19.566 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n '' ']' 2024-09-14T08:49:19.566 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ stress_write_image cluster2 mirror image_27 2024-09-14T08:49:19.566 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:49:19.566 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:49:19.567 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_27 2024-09-14T08:49:19.567 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T08:49:19.568 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local duration=7 2024-09-14T08:49:19.568 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set +e 2024-09-14T08:49:19.568 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ timeout 7s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_27 --debug-rbd=20 --debug-journaler=20 2024-09-14T08:49:19.790 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:100 IOs, 42676224 bytes 2024-09-14T08:49:19.870 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:200 IOs, 46736384 bytes 2024-09-14T08:49:20.007 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:300 IOs, 73573376 bytes 2024-09-14T08:49:20.243 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:400 IOs, 139203584 bytes 2024-09-14T08:49:20.393 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:500 IOs, 185594880 bytes 2024-09-14T08:49:20.705 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:600 IOs, 270808064 bytes 2024-09-14T08:49:20.868 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:700 IOs, 320104448 bytes 2024-09-14T08:49:21.080 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:800 IOs, 368898048 bytes 2024-09-14T08:49:21.189 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:900 IOs, 376270848 bytes 2024-09-14T08:49:21.383 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1000 IOs, 416446464 bytes 2024-09-14T08:49:21.572 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1100 IOs, 484894720 bytes 2024-09-14T08:49:21.711 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1200 IOs, 490368000 bytes 2024-09-14T08:49:21.811 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1300 IOs, 502296576 bytes 2024-09-14T08:49:22.056 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1400 IOs, 574595072 bytes 2024-09-14T08:49:22.155 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1500 IOs, 583780352 bytes 2024-09-14T08:49:22.317 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1600 IOs, 630240256 bytes 2024-09-14T08:49:22.603 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1700 IOs, 712109056 bytes 2024-09-14T08:49:22.803 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1800 IOs, 761628672 bytes 2024-09-14T08:49:22.983 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1900 IOs, 803119104 bytes 2024-09-14T08:49:23.171 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2000 IOs, 862495744 bytes 2024-09-14T08:49:23.330 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2100 IOs, 881208320 bytes 2024-09-14T08:49:23.526 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2200 IOs, 932705280 bytes 2024-09-14T08:49:23.641 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2300 IOs, 949919744 bytes 2024-09-14T08:49:23.842 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2400 IOs, 1010407424 bytes 2024-09-14T08:49:24.036 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2500 IOs, 1050857472 bytes 2024-09-14T08:49:24.171 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2600 IOs, 1074009088 bytes 2024-09-14T08:49:24.431 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2700 IOs, 1146614784 bytes 2024-09-14T08:49:24.506 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2800 IOs, 1147224064 bytes 2024-09-14T08:49:24.682 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2900 IOs, 1209253888 bytes 2024-09-14T08:49:24.926 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3000 IOs, 1262075904 bytes 2024-09-14T08:49:25.024 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3100 IOs, 1267963904 bytes 2024-09-14T08:49:25.295 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3200 IOs, 1350962176 bytes 2024-09-14T08:49:25.502 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3300 IOs, 1405301760 bytes 2024-09-14T08:49:25.634 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3400 IOs, 1439677440 bytes 2024-09-14T08:49:25.861 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3500 IOs, 1498064896 bytes 2024-09-14T08:49:26.046 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3600 IOs, 1539129344 bytes 2024-09-14T08:49:26.146 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3700 IOs, 1547767808 bytes 2024-09-14T08:49:26.286 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3800 IOs, 1573699584 bytes 2024-09-14T08:49:26.398 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3900 IOs, 1590745088 bytes 2024-09-14T08:49:26.482 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4000 IOs, 1606929408 bytes 2024-09-14T08:49:26.574 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ error_code=124 2024-09-14T08:49:26.574 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set -e 2024-09-14T08:49:26.574 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T08:49:26.574 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:49:26.574 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T08:49:26.574 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_28 2024-09-14T08:49:26.574 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image_and_enable_mirror cluster2 mirror image_28 snapshot 128M 2024-09-14T08:49:26.574 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:49:26.574 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:49:26.575 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:49:26.575 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:49:26.575 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_28 2024-09-14T08:49:26.575 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:49:26.575 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:49:26.575 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n snapshot ']' 2024-09-14T08:49:26.575 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:49:26.575 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image cluster2 mirror image_28 128M 2024-09-14T08:49:26.575 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:49:26.575 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:49:26.575 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:49:26.575 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:49:26.575 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_28 2024-09-14T08:49:26.575 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:49:26.576 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local size=128 2024-09-14T08:49:26.576 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n 128M ']' 2024-09-14T08:49:26.576 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ size=128M 2024-09-14T08:49:26.576 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:49:26.576 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_28 2024-09-14T08:49:26.623 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T08:49:26.623 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:49:26.623 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' image = image ']' 2024-09-14T08:49:26.623 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:49:26.623 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ enable_mirror cluster2 mirror image_28 snapshot 2024-09-14T08:49:26.623 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:49:26.623 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:49:26.623 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_28 2024-09-14T08:49:26.624 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:49:26.624 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_28 snapshot 2024-09-14T08:49:27.531 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Mirroring enabled 2024-09-14T08:49:27.537 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 info mirror/image_28 2024-09-14T08:49:27.581 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_28': 2024-09-14T08:49:27.581 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T08:49:27.582 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:49:27.582 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 1 2024-09-14T08:49:27.582 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 13bb431181db 2024-09-14T08:49:27.582 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.13bb431181db 2024-09-14T08:49:27.582 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:49:27.582 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering 2024-09-14T08:49:27.582 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:49:27.582 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:49:27.582 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:49:26 2024 2024-09-14T08:49:27.582 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:49:26 2024 2024-09-14T08:49:27.582 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:49:26 2024 2024-09-14T08:49:27.582 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:49:27.582 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:49:27.582 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: e5b5824c-8477-465c-bd16-4b286b9a4cc9 2024-09-14T08:49:27.583 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: true 2024-09-14T08:49:27.585 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n '' ']' 2024-09-14T08:49:27.585 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ stress_write_image cluster2 mirror image_28 2024-09-14T08:49:27.585 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:49:27.585 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:49:27.586 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_28 2024-09-14T08:49:27.586 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T08:49:27.587 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local duration=12 2024-09-14T08:49:27.588 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set +e 2024-09-14T08:49:27.588 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ timeout 12s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_28 --debug-rbd=20 --debug-journaler=20 2024-09-14T08:49:27.955 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:100 IOs, 21390336 bytes 2024-09-14T08:49:28.136 INFO:tasks.ceph.cluster2.osd.1.smithi164.stderr:2024-09-14T08:49:28.136+0000 7f90a8d57640 -1 reset not still connected to 0x55a2f8f50000 2024-09-14T08:49:28.158 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:200 IOs, 78156800 bytes 2024-09-14T08:49:28.242 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:300 IOs, 78754816 bytes 2024-09-14T08:49:28.310 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:400 IOs, 79385600 bytes 2024-09-14T08:49:28.507 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:500 IOs, 141809664 bytes 2024-09-14T08:49:28.769 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:600 IOs, 229238784 bytes 2024-09-14T08:49:29.132 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:700 IOs, 331026432 bytes 2024-09-14T08:49:29.315 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:800 IOs, 370127872 bytes 2024-09-14T08:49:29.460 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:900 IOs, 403647488 bytes 2024-09-14T08:49:29.652 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1000 IOs, 460262400 bytes 2024-09-14T08:49:29.829 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1100 IOs, 487813120 bytes 2024-09-14T08:49:30.004 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1200 IOs, 524526592 bytes 2024-09-14T08:49:30.245 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1300 IOs, 591893504 bytes 2024-09-14T08:49:30.411 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1400 IOs, 628894720 bytes 2024-09-14T08:49:30.504 INFO:tasks.ceph.cluster2.osd.1.smithi164.stderr:2024-09-14T08:49:30.504+0000 7f90a8d57640 -1 reset not still connected to 0x55a2f77431e0 2024-09-14T08:49:30.504 INFO:tasks.ceph.cluster2.osd.1.smithi164.stderr:2024-09-14T08:49:30.504+0000 7f90a8d57640 -1 reset not still connected to 0x55a2fa906d00 2024-09-14T08:49:30.540 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1500 IOs, 647998464 bytes 2024-09-14T08:49:30.712 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1600 IOs, 687741952 bytes 2024-09-14T08:49:30.804 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1700 IOs, 694733824 bytes 2024-09-14T08:49:30.942 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1800 IOs, 723653632 bytes 2024-09-14T08:49:31.021 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1900 IOs, 724265984 bytes 2024-09-14T08:49:31.102 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2000 IOs, 728186880 bytes 2024-09-14T08:49:31.198 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2100 IOs, 734698496 bytes 2024-09-14T08:49:31.411 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2200 IOs, 788087808 bytes 2024-09-14T08:49:31.572 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2300 IOs, 819718144 bytes 2024-09-14T08:49:31.730 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2400 IOs, 854562816 bytes 2024-09-14T08:49:31.849 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2500 IOs, 896608256 bytes 2024-09-14T08:49:32.150 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2600 IOs, 975804416 bytes 2024-09-14T08:49:32.380 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2700 IOs, 1033515008 bytes 2024-09-14T08:49:32.513 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2800 IOs, 1050699776 bytes 2024-09-14T08:49:32.769 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2900 IOs, 1101248512 bytes 2024-09-14T08:49:32.987 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3000 IOs, 1164792832 bytes 2024-09-14T08:49:33.119 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3100 IOs, 1187838976 bytes 2024-09-14T08:49:33.360 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3200 IOs, 1271941120 bytes 2024-09-14T08:49:33.688 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3300 IOs, 1362628608 bytes 2024-09-14T08:49:33.905 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3400 IOs, 1418176512 bytes 2024-09-14T08:49:34.172 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3500 IOs, 1499985920 bytes 2024-09-14T08:49:34.345 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3600 IOs, 1540077568 bytes 2024-09-14T08:49:34.481 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3700 IOs, 1572831232 bytes 2024-09-14T08:49:34.665 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3800 IOs, 1611703296 bytes 2024-09-14T08:49:34.846 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3900 IOs, 1651904512 bytes 2024-09-14T08:49:35.017 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4000 IOs, 1688145920 bytes 2024-09-14T08:49:35.113 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4100 IOs, 1697504256 bytes 2024-09-14T08:49:35.396 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4200 IOs, 1783614464 bytes 2024-09-14T08:49:35.570 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4300 IOs, 1829766144 bytes 2024-09-14T08:49:35.700 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4400 IOs, 1849112576 bytes 2024-09-14T08:49:35.864 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4500 IOs, 1885964288 bytes 2024-09-14T08:49:36.149 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4600 IOs, 1973926912 bytes 2024-09-14T08:49:36.382 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4700 IOs, 2034718720 bytes 2024-09-14T08:49:36.562 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4800 IOs, 2086808576 bytes 2024-09-14T08:49:36.768 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4900 IOs, 2124672000 bytes 2024-09-14T08:49:36.881 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5000 IOs, 2150335488 bytes 2024-09-14T08:49:36.986 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5100 IOs, 2182515712 bytes 2024-09-14T08:49:37.277 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5200 IOs, 2239370240 bytes 2024-09-14T08:49:37.420 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5300 IOs, 2269828096 bytes 2024-09-14T08:49:37.617 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5400 IOs, 2318263296 bytes 2024-09-14T08:49:37.836 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5500 IOs, 2378858496 bytes 2024-09-14T08:49:37.976 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5600 IOs, 2406214656 bytes 2024-09-14T08:49:38.169 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5700 IOs, 2448625664 bytes 2024-09-14T08:49:38.392 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5800 IOs, 2504845312 bytes 2024-09-14T08:49:38.559 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5900 IOs, 2543705088 bytes 2024-09-14T08:49:38.853 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6000 IOs, 2631380992 bytes 2024-09-14T08:49:39.020 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6100 IOs, 2666777600 bytes 2024-09-14T08:49:39.206 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6200 IOs, 2711522304 bytes 2024-09-14T08:49:39.337 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6300 IOs, 2733142016 bytes 2024-09-14T08:49:39.526 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6400 IOs, 2783650816 bytes 2024-09-14T08:49:39.593 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ error_code=124 2024-09-14T08:49:39.593 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set -e 2024-09-14T08:49:39.593 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T08:49:39.593 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:49:39.593 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T08:49:39.593 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_29 2024-09-14T08:49:39.593 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image_and_enable_mirror cluster2 mirror image_29 snapshot 128M 2024-09-14T08:49:39.593 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:49:39.593 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:49:39.593 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:49:39.594 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:49:39.594 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_29 2024-09-14T08:49:39.594 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:49:39.594 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:49:39.594 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n snapshot ']' 2024-09-14T08:49:39.594 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:49:39.594 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image cluster2 mirror image_29 128M 2024-09-14T08:49:39.594 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:49:39.594 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:49:39.594 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:49:39.594 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:49:39.594 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_29 2024-09-14T08:49:39.594 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:49:39.594 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local size=128 2024-09-14T08:49:39.595 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n 128M ']' 2024-09-14T08:49:39.595 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ size=128M 2024-09-14T08:49:39.595 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:49:39.595 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_29 2024-09-14T08:49:39.641 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T08:49:39.642 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:49:39.642 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' image = image ']' 2024-09-14T08:49:39.642 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:49:39.642 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ enable_mirror cluster2 mirror image_29 snapshot 2024-09-14T08:49:39.642 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:49:39.642 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:49:39.642 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_29 2024-09-14T08:49:39.642 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:49:39.642 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_29 snapshot 2024-09-14T08:49:40.538 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Mirroring enabled 2024-09-14T08:49:40.543 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 info mirror/image_29 2024-09-14T08:49:40.583 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_29': 2024-09-14T08:49:40.583 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T08:49:40.583 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:49:40.583 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 1 2024-09-14T08:49:40.583 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 13c78158ca42 2024-09-14T08:49:40.583 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.13c78158ca42 2024-09-14T08:49:40.583 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:49:40.583 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering 2024-09-14T08:49:40.583 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:49:40.583 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:49:40.584 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:49:39 2024 2024-09-14T08:49:40.584 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:49:39 2024 2024-09-14T08:49:40.584 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:49:39 2024 2024-09-14T08:49:40.584 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:49:40.584 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:49:40.584 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: 50c4f65a-9027-4112-bc18-18bc2d737492 2024-09-14T08:49:40.584 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: true 2024-09-14T08:49:40.586 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n '' ']' 2024-09-14T08:49:40.587 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ stress_write_image cluster2 mirror image_29 2024-09-14T08:49:40.587 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:49:40.587 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:49:40.587 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_29 2024-09-14T08:49:40.587 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T08:49:40.588 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local duration=11 2024-09-14T08:49:40.588 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set +e 2024-09-14T08:49:40.588 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ timeout 11s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_29 --debug-rbd=20 --debug-journaler=20 2024-09-14T08:49:40.928 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:100 IOs, 98567168 bytes 2024-09-14T08:49:41.081 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:200 IOs, 133099520 bytes 2024-09-14T08:49:41.286 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:300 IOs, 186938368 bytes 2024-09-14T08:49:41.623 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:400 IOs, 310907904 bytes 2024-09-14T08:49:41.831 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:500 IOs, 360933376 bytes 2024-09-14T08:49:41.941 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:600 IOs, 368106496 bytes 2024-09-14T08:49:42.020 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:700 IOs, 377918464 bytes 2024-09-14T08:49:42.168 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:800 IOs, 410423296 bytes 2024-09-14T08:49:42.349 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:900 IOs, 442971136 bytes 2024-09-14T08:49:42.572 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1000 IOs, 503523328 bytes 2024-09-14T08:49:42.652 INFO:tasks.ceph.cluster2.osd.2.smithi164.stderr:2024-09-14T08:49:42.651+0000 7fe32301f640 -1 reset not still connected to 0x55e755eb7790 2024-09-14T08:49:42.652 INFO:tasks.ceph.cluster2.osd.2.smithi164.stderr:2024-09-14T08:49:42.651+0000 7fe32301f640 -1 reset not still connected to 0x55e7573ad860 2024-09-14T08:49:42.832 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1100 IOs, 575215616 bytes 2024-09-14T08:49:42.972 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1200 IOs, 612774912 bytes 2024-09-14T08:49:43.209 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1300 IOs, 682259456 bytes 2024-09-14T08:49:43.383 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1400 IOs, 718045184 bytes 2024-09-14T08:49:43.477 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1500 IOs, 724698112 bytes 2024-09-14T08:49:43.554 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1600 IOs, 725335040 bytes 2024-09-14T08:49:43.628 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1700 IOs, 725916672 bytes 2024-09-14T08:49:43.951 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1800 IOs, 830999552 bytes 2024-09-14T08:49:44.046 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1900 IOs, 847558656 bytes 2024-09-14T08:49:44.306 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2000 IOs, 923343872 bytes 2024-09-14T08:49:44.489 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2100 IOs, 959497216 bytes 2024-09-14T08:49:44.558 INFO:tasks.ceph.cluster2.osd.2.smithi164.stderr:2024-09-14T08:49:44.558+0000 7fe32301f640 -1 reset not still connected to 0x55e760ed8ea0 2024-09-14T08:49:44.600 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2200 IOs, 972115968 bytes 2024-09-14T08:49:44.705 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2300 IOs, 987295744 bytes 2024-09-14T08:49:44.875 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2400 IOs, 1041138688 bytes 2024-09-14T08:49:44.997 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2500 IOs, 1052288000 bytes 2024-09-14T08:49:45.167 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2600 IOs, 1090924544 bytes 2024-09-14T08:49:45.345 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2700 IOs, 1134960640 bytes 2024-09-14T08:49:45.465 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2800 IOs, 1151338496 bytes 2024-09-14T08:49:45.598 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2900 IOs, 1171768320 bytes 2024-09-14T08:49:45.765 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3000 IOs, 1210944512 bytes 2024-09-14T08:49:46.000 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3100 IOs, 1278571520 bytes 2024-09-14T08:49:46.087 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3200 IOs, 1282989056 bytes 2024-09-14T08:49:46.267 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3300 IOs, 1325471744 bytes 2024-09-14T08:49:46.432 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3400 IOs, 1365602304 bytes 2024-09-14T08:49:46.648 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3500 IOs, 1421672448 bytes 2024-09-14T08:49:46.898 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3600 IOs, 1499292672 bytes 2024-09-14T08:49:47.182 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3700 IOs, 1584663552 bytes 2024-09-14T08:49:47.347 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3800 IOs, 1630742528 bytes 2024-09-14T08:49:47.492 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3900 IOs, 1647391744 bytes 2024-09-14T08:49:47.568 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4000 IOs, 1648031744 bytes 2024-09-14T08:49:47.755 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4100 IOs, 1706998784 bytes 2024-09-14T08:49:47.914 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4200 IOs, 1734551552 bytes 2024-09-14T08:49:48.057 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4300 IOs, 1757983744 bytes 2024-09-14T08:49:48.267 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4400 IOs, 1811375104 bytes 2024-09-14T08:49:48.384 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4500 IOs, 1827559424 bytes 2024-09-14T08:49:48.570 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4600 IOs, 1873627136 bytes 2024-09-14T08:49:48.722 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4700 IOs, 1908235264 bytes 2024-09-14T08:49:48.805 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4800 IOs, 1908837376 bytes 2024-09-14T08:49:48.896 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4900 IOs, 1916237824 bytes 2024-09-14T08:49:49.041 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5000 IOs, 1948640256 bytes 2024-09-14T08:49:49.246 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5100 IOs, 2004058112 bytes 2024-09-14T08:49:49.413 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5200 IOs, 2035945472 bytes 2024-09-14T08:49:49.609 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5300 IOs, 2085133312 bytes 2024-09-14T08:49:49.703 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5400 IOs, 2095623168 bytes 2024-09-14T08:49:49.895 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5500 IOs, 2143526912 bytes 2024-09-14T08:49:50.032 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5600 IOs, 2170062848 bytes 2024-09-14T08:49:50.215 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5700 IOs, 2217358336 bytes 2024-09-14T08:49:50.456 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5800 IOs, 2300114944 bytes 2024-09-14T08:49:50.647 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5900 IOs, 2330949632 bytes 2024-09-14T08:49:50.741 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6000 IOs, 2352858112 bytes 2024-09-14T08:49:50.953 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6100 IOs, 2404345856 bytes 2024-09-14T08:49:51.116 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6200 IOs, 2427935744 bytes 2024-09-14T08:49:51.319 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6300 IOs, 2484152320 bytes 2024-09-14T08:49:51.464 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6400 IOs, 2506050560 bytes 2024-09-14T08:49:51.595 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ error_code=124 2024-09-14T08:49:51.595 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set -e 2024-09-14T08:49:51.596 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T08:49:51.596 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:49:51.596 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T08:49:51.596 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_30 2024-09-14T08:49:51.596 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image_and_enable_mirror cluster2 mirror image_30 snapshot 128M 2024-09-14T08:49:51.596 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:49:51.596 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:49:51.596 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:49:51.596 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:49:51.596 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_30 2024-09-14T08:49:51.596 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:49:51.596 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:49:51.596 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n snapshot ']' 2024-09-14T08:49:51.597 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:49:51.597 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image cluster2 mirror image_30 128M 2024-09-14T08:49:51.597 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:49:51.597 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:49:51.598 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:49:51.598 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:49:51.598 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_30 2024-09-14T08:49:51.598 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:49:51.598 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local size=128 2024-09-14T08:49:51.598 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n 128M ']' 2024-09-14T08:49:51.598 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ size=128M 2024-09-14T08:49:51.598 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:49:51.598 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_30 2024-09-14T08:49:51.650 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T08:49:51.650 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:49:51.650 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' image = image ']' 2024-09-14T08:49:51.650 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:49:51.650 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ enable_mirror cluster2 mirror image_30 snapshot 2024-09-14T08:49:51.650 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:49:51.650 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:49:51.650 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_30 2024-09-14T08:49:51.650 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:49:51.651 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_30 snapshot 2024-09-14T08:49:52.559 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Mirroring enabled 2024-09-14T08:49:52.565 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 info mirror/image_30 2024-09-14T08:49:52.607 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_30': 2024-09-14T08:49:52.607 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T08:49:52.607 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:49:52.607 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 1 2024-09-14T08:49:52.607 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 13d3c949a71e 2024-09-14T08:49:52.607 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.13d3c949a71e 2024-09-14T08:49:52.608 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:49:52.608 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering 2024-09-14T08:49:52.608 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:49:52.608 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:49:52.608 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:49:51 2024 2024-09-14T08:49:52.608 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:49:51 2024 2024-09-14T08:49:52.608 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:49:51 2024 2024-09-14T08:49:52.608 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:49:52.608 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:49:52.608 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: 4e345c40-e13b-4117-8365-22395b48e4b8 2024-09-14T08:49:52.608 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: true 2024-09-14T08:49:52.610 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n '' ']' 2024-09-14T08:49:52.611 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ stress_write_image cluster2 mirror image_30 2024-09-14T08:49:52.611 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:49:52.611 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:49:52.611 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_30 2024-09-14T08:49:52.611 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T08:49:52.613 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local duration=12 2024-09-14T08:49:52.613 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set +e 2024-09-14T08:49:52.613 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ timeout 12s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_30 --debug-rbd=20 --debug-journaler=20 2024-09-14T08:49:52.937 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:100 IOs, 87911424 bytes 2024-09-14T08:49:53.092 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:200 IOs, 116245504 bytes 2024-09-14T08:49:53.378 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:300 IOs, 214436864 bytes 2024-09-14T08:49:53.578 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:400 IOs, 261695488 bytes 2024-09-14T08:49:53.717 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:500 IOs, 272274432 bytes 2024-09-14T08:49:53.813 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:600 IOs, 282455040 bytes 2024-09-14T08:49:54.029 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:700 IOs, 342126592 bytes 2024-09-14T08:49:54.137 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:800 IOs, 372502528 bytes 2024-09-14T08:49:54.427 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:900 IOs, 452912128 bytes 2024-09-14T08:49:54.643 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1000 IOs, 494121984 bytes 2024-09-14T08:49:54.801 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1100 IOs, 536065024 bytes 2024-09-14T08:49:54.897 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1200 IOs, 538288128 bytes 2024-09-14T08:49:54.996 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1300 IOs, 550960128 bytes 2024-09-14T08:49:55.131 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1400 IOs, 575773696 bytes 2024-09-14T08:49:55.325 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1500 IOs, 628194304 bytes 2024-09-14T08:49:55.472 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1600 IOs, 667205632 bytes 2024-09-14T08:49:55.672 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1700 IOs, 701380608 bytes 2024-09-14T08:49:55.860 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1800 IOs, 751863808 bytes 2024-09-14T08:49:55.962 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1900 IOs, 761662464 bytes 2024-09-14T08:49:56.178 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2000 IOs, 814514176 bytes 2024-09-14T08:49:56.442 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2100 IOs, 891185152 bytes 2024-09-14T08:49:56.586 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2200 IOs, 913833984 bytes 2024-09-14T08:49:56.749 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2300 IOs, 968053760 bytes 2024-09-14T08:49:56.976 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2400 IOs, 1011551232 bytes 2024-09-14T08:49:57.166 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2500 IOs, 1056979968 bytes 2024-09-14T08:49:57.422 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2600 IOs, 1130598400 bytes 2024-09-14T08:49:57.623 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2700 IOs, 1180974080 bytes 2024-09-14T08:49:57.735 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2800 IOs, 1203161088 bytes 2024-09-14T08:49:57.978 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2900 IOs, 1294229504 bytes 2024-09-14T08:49:58.201 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3000 IOs, 1346457600 bytes 2024-09-14T08:49:58.257 INFO:tasks.ceph.cluster2.osd.1.smithi164.stderr:2024-09-14T08:49:58.255+0000 7f90a8d57640 -1 reset not still connected to 0x55a2fa9a1d40 2024-09-14T08:49:58.317 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3100 IOs, 1347091456 bytes 2024-09-14T08:49:58.481 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3200 IOs, 1385189376 bytes 2024-09-14T08:49:58.750 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3300 IOs, 1458549760 bytes 2024-09-14T08:49:58.919 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3400 IOs, 1511097344 bytes 2024-09-14T08:49:59.153 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3500 IOs, 1569769472 bytes 2024-09-14T08:49:59.382 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3600 IOs, 1620106240 bytes 2024-09-14T08:49:59.560 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3700 IOs, 1665530880 bytes 2024-09-14T08:49:59.801 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3800 IOs, 1726710784 bytes 2024-09-14T08:49:59.968 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3900 IOs, 1764861952 bytes 2024-09-14T08:50:00.101 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4000 IOs, 1789327360 bytes 2024-09-14T08:50:00.139 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.0 2024-09-14T08:50:00.139 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Sent signal 15 2024-09-14T08:50:00.139 INFO:tasks.rbd_mirror_thrash:waiting for 8 secs before reviving daemons 2024-09-14T08:50:00.141 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi164.stderr:2024-09-14T08:50:00.140+0000 7fd6de12a640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 (PID: 45262) UID: 1000 2024-09-14T08:50:00.231 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4100 IOs, 1810611200 bytes 2024-09-14T08:50:00.360 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4200 IOs, 1824326656 bytes 2024-09-14T08:50:00.565 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4300 IOs, 1876612096 bytes 2024-09-14T08:50:00.794 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4400 IOs, 1937874944 bytes 2024-09-14T08:50:00.889 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4500 IOs, 1946684416 bytes 2024-09-14T08:50:00.977 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4600 IOs, 1951654912 bytes 2024-09-14T08:50:01.206 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4700 IOs, 2032680960 bytes 2024-09-14T08:50:01.433 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4800 IOs, 2074579968 bytes 2024-09-14T08:50:01.681 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4900 IOs, 2146819072 bytes 2024-09-14T08:50:01.932 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5000 IOs, 2216884224 bytes 2024-09-14T08:50:02.114 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5100 IOs, 2284336128 bytes 2024-09-14T08:50:02.254 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5200 IOs, 2285725696 bytes 2024-09-14T08:50:02.361 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5300 IOs, 2299513856 bytes 2024-09-14T08:50:02.505 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5400 IOs, 2327897088 bytes 2024-09-14T08:50:02.755 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5500 IOs, 2415459328 bytes 2024-09-14T08:50:02.982 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5600 IOs, 2458544128 bytes 2024-09-14T08:50:03.074 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5700 IOs, 2497251328 bytes 2024-09-14T08:50:03.304 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5800 IOs, 2538143744 bytes 2024-09-14T08:50:03.445 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5900 IOs, 2559150080 bytes 2024-09-14T08:50:03.644 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6000 IOs, 2618696704 bytes 2024-09-14T08:50:03.967 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6100 IOs, 2702803968 bytes 2024-09-14T08:50:04.100 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6200 IOs, 2727690240 bytes 2024-09-14T08:50:04.341 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6300 IOs, 2797457408 bytes 2024-09-14T08:50:04.504 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6400 IOs, 2850522112 bytes 2024-09-14T08:50:04.619 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ error_code=124 2024-09-14T08:50:04.619 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set -e 2024-09-14T08:50:04.619 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T08:50:04.619 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:50:04.619 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T08:50:04.619 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_31 2024-09-14T08:50:04.619 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image_and_enable_mirror cluster2 mirror image_31 snapshot 128M 2024-09-14T08:50:04.619 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:50:04.619 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:50:04.619 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:50:04.619 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:50:04.620 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_31 2024-09-14T08:50:04.620 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:50:04.620 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:50:04.620 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n snapshot ']' 2024-09-14T08:50:04.620 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:50:04.620 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image cluster2 mirror image_31 128M 2024-09-14T08:50:04.620 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:50:04.620 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:50:04.620 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:50:04.620 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:50:04.620 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_31 2024-09-14T08:50:04.620 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:50:04.620 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local size=128 2024-09-14T08:50:04.621 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n 128M ']' 2024-09-14T08:50:04.621 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ size=128M 2024-09-14T08:50:04.621 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:50:04.621 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_31 2024-09-14T08:50:04.675 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T08:50:04.676 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:50:04.676 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' image = image ']' 2024-09-14T08:50:04.676 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:50:04.676 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ enable_mirror cluster2 mirror image_31 snapshot 2024-09-14T08:50:04.676 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:50:04.676 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:50:04.676 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_31 2024-09-14T08:50:04.676 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:50:04.676 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_31 snapshot 2024-09-14T08:50:05.596 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Mirroring enabled 2024-09-14T08:50:05.602 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 info mirror/image_31 2024-09-14T08:50:05.641 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_31': 2024-09-14T08:50:05.641 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T08:50:05.641 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:50:05.641 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 1 2024-09-14T08:50:05.642 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 13decf3be216 2024-09-14T08:50:05.642 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.13decf3be216 2024-09-14T08:50:05.642 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:50:05.642 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering 2024-09-14T08:50:05.642 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:50:05.642 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:50:05.642 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:50:04 2024 2024-09-14T08:50:05.642 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:50:04 2024 2024-09-14T08:50:05.642 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:50:04 2024 2024-09-14T08:50:05.642 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:50:05.642 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:50:05.642 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: ca4b7bbf-4e51-4087-ac80-e20897556ab1 2024-09-14T08:50:05.642 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: true 2024-09-14T08:50:05.645 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n '' ']' 2024-09-14T08:50:05.645 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ stress_write_image cluster2 mirror image_31 2024-09-14T08:50:05.645 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:50:05.645 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:50:05.645 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_31 2024-09-14T08:50:05.645 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T08:50:05.646 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local duration=9 2024-09-14T08:50:05.646 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set +e 2024-09-14T08:50:05.647 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ timeout 9s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_31 --debug-rbd=20 --debug-journaler=20 2024-09-14T08:50:05.840 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:100 IOs, 30301184 bytes 2024-09-14T08:50:06.045 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:200 IOs, 83668992 bytes 2024-09-14T08:50:06.318 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:300 IOs, 159380480 bytes 2024-09-14T08:50:06.524 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:400 IOs, 213364736 bytes 2024-09-14T08:50:06.830 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:500 IOs, 305977344 bytes 2024-09-14T08:50:07.024 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:600 IOs, 355325952 bytes 2024-09-14T08:50:07.154 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:700 IOs, 392893440 bytes 2024-09-14T08:50:07.425 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:800 IOs, 448876544 bytes 2024-09-14T08:50:07.544 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:900 IOs, 477393920 bytes 2024-09-14T08:50:07.741 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1000 IOs, 512133120 bytes 2024-09-14T08:50:07.962 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1100 IOs, 553046016 bytes 2024-09-14T08:50:08.141 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.0 2024-09-14T08:50:08.141 INFO:teuthology.orchestra.run:waiting for 600 2024-09-14T08:50:08.141 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.0 2024-09-14T08:50:08.141 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Restarting daemon 2024-09-14T08:50:08.142 DEBUG:teuthology.orchestra.run.smithi164:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 2024-09-14T08:50:08.145 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Started 2024-09-14T08:50:08.145 INFO:tasks.rbd_mirror_thrash:waiting for 95 secs before thrashing 2024-09-14T08:50:08.216 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1200 IOs, 619311104 bytes 2024-09-14T08:50:08.414 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1300 IOs, 662677504 bytes 2024-09-14T08:50:08.517 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1400 IOs, 674314240 bytes 2024-09-14T08:50:08.735 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1500 IOs, 724570112 bytes 2024-09-14T08:50:08.935 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1600 IOs, 804248576 bytes 2024-09-14T08:50:09.131 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1700 IOs, 818086912 bytes 2024-09-14T08:50:09.341 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1800 IOs, 871182336 bytes 2024-09-14T08:50:09.465 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1900 IOs, 908873728 bytes 2024-09-14T08:50:09.678 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2000 IOs, 962505728 bytes 2024-09-14T08:50:09.892 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2100 IOs, 1015420928 bytes 2024-09-14T08:50:10.139 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2200 IOs, 1073265664 bytes 2024-09-14T08:50:10.280 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2300 IOs, 1085776896 bytes 2024-09-14T08:50:10.496 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2400 IOs, 1123099648 bytes 2024-09-14T08:50:10.680 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2500 IOs, 1172753408 bytes 2024-09-14T08:50:10.895 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2600 IOs, 1221429248 bytes 2024-09-14T08:50:11.064 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2700 IOs, 1287952384 bytes 2024-09-14T08:50:11.313 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2800 IOs, 1325082624 bytes 2024-09-14T08:50:11.486 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2900 IOs, 1366223872 bytes 2024-09-14T08:50:11.581 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3000 IOs, 1371459584 bytes 2024-09-14T08:50:11.786 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3100 IOs, 1424874496 bytes 2024-09-14T08:50:11.896 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3200 IOs, 1441080320 bytes 2024-09-14T08:50:12.037 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3300 IOs, 1465008128 bytes 2024-09-14T08:50:12.126 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3400 IOs, 1471097856 bytes 2024-09-14T08:50:12.291 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3500 IOs, 1515961344 bytes 2024-09-14T08:50:12.534 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3600 IOs, 1579205632 bytes 2024-09-14T08:50:12.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3700 IOs, 1617735680 bytes 2024-09-14T08:50:12.902 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3800 IOs, 1656671232 bytes 2024-09-14T08:50:13.059 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3900 IOs, 1688875008 bytes 2024-09-14T08:50:13.270 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4000 IOs, 1739215872 bytes 2024-09-14T08:50:13.392 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4100 IOs, 1759149056 bytes 2024-09-14T08:50:13.593 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4200 IOs, 1807039488 bytes 2024-09-14T08:50:13.738 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4300 IOs, 1835227136 bytes 2024-09-14T08:50:14.075 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4400 IOs, 1939816448 bytes 2024-09-14T08:50:14.251 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4500 IOs, 1980564480 bytes 2024-09-14T08:50:14.371 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4600 IOs, 1998777344 bytes 2024-09-14T08:50:14.611 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4700 IOs, 2060322816 bytes 2024-09-14T08:50:14.657 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ error_code=124 2024-09-14T08:50:14.657 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set -e 2024-09-14T08:50:14.657 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T08:50:14.657 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:50:14.658 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T08:50:14.658 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_32 2024-09-14T08:50:14.658 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image_and_enable_mirror cluster2 mirror image_32 snapshot 128M 2024-09-14T08:50:14.658 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:50:14.658 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:50:14.658 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:50:14.658 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:50:14.658 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_32 2024-09-14T08:50:14.658 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:50:14.658 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:50:14.658 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n snapshot ']' 2024-09-14T08:50:14.658 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:50:14.658 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image cluster2 mirror image_32 128M 2024-09-14T08:50:14.658 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:50:14.659 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:50:14.659 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:50:14.659 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:50:14.659 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_32 2024-09-14T08:50:14.659 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:50:14.659 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local size=128 2024-09-14T08:50:14.659 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n 128M ']' 2024-09-14T08:50:14.659 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ size=128M 2024-09-14T08:50:14.659 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:50:14.659 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_32 2024-09-14T08:50:14.727 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T08:50:14.727 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:50:14.727 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' image = image ']' 2024-09-14T08:50:14.727 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:50:14.728 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ enable_mirror cluster2 mirror image_32 snapshot 2024-09-14T08:50:14.728 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:50:14.728 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:50:14.728 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_32 2024-09-14T08:50:14.728 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:50:14.728 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_32 snapshot 2024-09-14T08:50:15.606 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Mirroring enabled 2024-09-14T08:50:15.612 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 info mirror/image_32 2024-09-14T08:50:15.653 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_32': 2024-09-14T08:50:15.653 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T08:50:15.653 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:50:15.653 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 1 2024-09-14T08:50:15.653 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 13f09f6248ce 2024-09-14T08:50:15.653 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.13f09f6248ce 2024-09-14T08:50:15.653 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:50:15.653 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering 2024-09-14T08:50:15.653 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:50:15.653 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:50:15.653 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:50:14 2024 2024-09-14T08:50:15.654 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:50:14 2024 2024-09-14T08:50:15.654 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:50:14 2024 2024-09-14T08:50:15.654 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:50:15.654 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:50:15.654 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: 7da0399c-f6eb-473e-a82d-0b57a2ea8a70 2024-09-14T08:50:15.654 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: true 2024-09-14T08:50:15.656 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n '' ']' 2024-09-14T08:50:15.656 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ stress_write_image cluster2 mirror image_32 2024-09-14T08:50:15.656 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:50:15.656 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:50:15.657 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_32 2024-09-14T08:50:15.657 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T08:50:15.658 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local duration=13 2024-09-14T08:50:15.658 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set +e 2024-09-14T08:50:15.658 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ timeout 13s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_32 --debug-rbd=20 --debug-journaler=20 2024-09-14T08:50:15.877 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:100 IOs, 39987200 bytes 2024-09-14T08:50:15.961 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:200 IOs, 45025280 bytes 2024-09-14T08:50:16.082 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:300 IOs, 61829120 bytes 2024-09-14T08:50:16.252 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:400 IOs, 111135744 bytes 2024-09-14T08:50:16.573 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:500 IOs, 143356928 bytes 2024-09-14T08:50:16.783 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:600 IOs, 193520640 bytes 2024-09-14T08:50:16.908 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:700 IOs, 215815168 bytes 2024-09-14T08:50:17.095 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:800 IOs, 263316480 bytes 2024-09-14T08:50:17.326 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:900 IOs, 322640896 bytes 2024-09-14T08:50:17.560 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1000 IOs, 384239616 bytes 2024-09-14T08:50:17.646 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1100 IOs, 384854016 bytes 2024-09-14T08:50:17.843 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1200 IOs, 433525760 bytes 2024-09-14T08:50:17.952 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1300 IOs, 447590400 bytes 2024-09-14T08:50:18.181 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1400 IOs, 508513280 bytes 2024-09-14T08:50:18.489 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1500 IOs, 608393216 bytes 2024-09-14T08:50:18.623 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1600 IOs, 628408320 bytes 2024-09-14T08:50:18.754 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1700 IOs, 648671232 bytes 2024-09-14T08:50:18.873 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1800 IOs, 666729472 bytes 2024-09-14T08:50:19.090 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1900 IOs, 726085632 bytes 2024-09-14T08:50:19.240 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2000 IOs, 754776064 bytes 2024-09-14T08:50:19.410 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2100 IOs, 789498880 bytes 2024-09-14T08:50:19.633 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2200 IOs, 847374336 bytes 2024-09-14T08:50:19.771 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2300 IOs, 872750080 bytes 2024-09-14T08:50:19.931 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2400 IOs, 909752320 bytes 2024-09-14T08:50:20.142 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2500 IOs, 959340544 bytes 2024-09-14T08:50:20.412 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2600 IOs, 1042612224 bytes 2024-09-14T08:50:20.522 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2700 IOs, 1053093888 bytes 2024-09-14T08:50:20.706 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2800 IOs, 1095499776 bytes 2024-09-14T08:50:20.851 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2900 IOs, 1133965312 bytes 2024-09-14T08:50:21.086 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3000 IOs, 1190535168 bytes 2024-09-14T08:50:21.389 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3100 IOs, 1273745408 bytes 2024-09-14T08:50:21.539 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3200 IOs, 1319741440 bytes 2024-09-14T08:50:21.701 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3300 IOs, 1339887616 bytes 2024-09-14T08:50:21.840 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3400 IOs, 1363058688 bytes 2024-09-14T08:50:22.010 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3500 IOs, 1394617344 bytes 2024-09-14T08:50:22.224 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3600 IOs, 1450514432 bytes 2024-09-14T08:50:22.339 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3700 IOs, 1466806272 bytes 2024-09-14T08:50:22.479 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3800 IOs, 1500898304 bytes 2024-09-14T08:50:22.703 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3900 IOs, 1554959360 bytes 2024-09-14T08:50:22.863 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4000 IOs, 1589791744 bytes 2024-09-14T08:50:23.213 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4100 IOs, 1708542976 bytes 2024-09-14T08:50:23.396 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4200 IOs, 1744604160 bytes 2024-09-14T08:50:23.543 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4300 IOs, 1778349056 bytes 2024-09-14T08:50:23.708 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4400 IOs, 1814719488 bytes 2024-09-14T08:50:23.833 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4500 IOs, 1833579520 bytes 2024-09-14T08:50:24.001 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4600 IOs, 1872595968 bytes 2024-09-14T08:50:24.212 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4700 IOs, 1927060480 bytes 2024-09-14T08:50:24.332 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4800 IOs, 1944616960 bytes 2024-09-14T08:50:24.532 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4900 IOs, 1991418880 bytes 2024-09-14T08:50:24.720 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5000 IOs, 2038876160 bytes 2024-09-14T08:50:25.025 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5100 IOs, 2139067392 bytes 2024-09-14T08:50:25.200 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5200 IOs, 2170056704 bytes 2024-09-14T08:50:25.376 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5300 IOs, 2231305216 bytes 2024-09-14T08:50:25.610 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5400 IOs, 2275445760 bytes 2024-09-14T08:50:25.918 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.4231687564519022) > weight (0.25) 2024-09-14T08:50:25.918 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.1 2024-09-14T08:50:25.919 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Sent signal 15 2024-09-14T08:50:25.919 INFO:tasks.rbd_mirror_thrash:waiting for 2 secs before reviving daemons 2024-09-14T08:50:25.920 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi164.stderr:2024-09-14T08:50:25.919+0000 7f1337ba9640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 (PID: 42816) UID: 1000 2024-09-14T08:50:25.943 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5500 IOs, 2369184768 bytes 2024-09-14T08:50:26.042 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5600 IOs, 2375937024 bytes 2024-09-14T08:50:26.260 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5700 IOs, 2429849600 bytes 2024-09-14T08:50:26.407 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5800 IOs, 2458723328 bytes 2024-09-14T08:50:26.578 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5900 IOs, 2495984640 bytes 2024-09-14T08:50:26.738 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6000 IOs, 2531536896 bytes 2024-09-14T08:50:26.865 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6100 IOs, 2551148544 bytes 2024-09-14T08:50:27.134 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6200 IOs, 2624719872 bytes 2024-09-14T08:50:27.268 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6300 IOs, 2647502848 bytes 2024-09-14T08:50:27.372 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6400 IOs, 2674882560 bytes 2024-09-14T08:50:27.579 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6500 IOs, 2710366208 bytes 2024-09-14T08:50:27.699 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6600 IOs, 2718593024 bytes 2024-09-14T08:50:27.910 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6700 IOs, 2786156544 bytes 2024-09-14T08:50:27.920 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.1 2024-09-14T08:50:27.920 INFO:teuthology.orchestra.run:waiting for 600 2024-09-14T08:50:27.920 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.1 2024-09-14T08:50:27.920 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Restarting daemon 2024-09-14T08:50:27.921 DEBUG:teuthology.orchestra.run.smithi164:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 2024-09-14T08:50:27.923 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Started 2024-09-14T08:50:27.923 INFO:tasks.rbd_mirror_thrash:waiting for 79 secs before thrashing 2024-09-14T08:50:28.187 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6800 IOs, 2841938944 bytes 2024-09-14T08:50:28.415 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6900 IOs, 2894588928 bytes 2024-09-14T08:50:28.610 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:7000 IOs, 2947725312 bytes 2024-09-14T08:50:28.664 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ error_code=124 2024-09-14T08:50:28.664 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set -e 2024-09-14T08:50:28.664 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T08:50:28.664 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:50:28.664 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T08:50:28.664 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_33 2024-09-14T08:50:28.664 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image_and_enable_mirror cluster2 mirror image_33 snapshot 128M 2024-09-14T08:50:28.664 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:50:28.664 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:50:28.665 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:50:28.665 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:50:28.665 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_33 2024-09-14T08:50:28.665 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:50:28.665 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:50:28.665 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n snapshot ']' 2024-09-14T08:50:28.666 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:50:28.666 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image cluster2 mirror image_33 128M 2024-09-14T08:50:28.666 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:50:28.666 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:50:28.666 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:50:28.666 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:50:28.666 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_33 2024-09-14T08:50:28.666 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:50:28.666 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local size=128 2024-09-14T08:50:28.666 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n 128M ']' 2024-09-14T08:50:28.667 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ size=128M 2024-09-14T08:50:28.667 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:50:28.667 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_33 2024-09-14T08:50:28.713 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T08:50:28.713 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:50:28.713 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' image = image ']' 2024-09-14T08:50:28.713 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:50:28.714 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ enable_mirror cluster2 mirror image_33 snapshot 2024-09-14T08:50:28.714 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:50:28.714 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:50:28.714 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_33 2024-09-14T08:50:28.714 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:50:28.714 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_33 snapshot 2024-09-14T08:50:29.537 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Mirroring enabled 2024-09-14T08:50:29.545 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 info mirror/image_33 2024-09-14T08:50:29.586 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_33': 2024-09-14T08:50:29.586 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T08:50:29.587 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:50:29.587 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 1 2024-09-14T08:50:29.587 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 14045f66cdbf 2024-09-14T08:50:29.587 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.14045f66cdbf 2024-09-14T08:50:29.587 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:50:29.587 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering 2024-09-14T08:50:29.587 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:50:29.587 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:50:29.587 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:50:28 2024 2024-09-14T08:50:29.587 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:50:28 2024 2024-09-14T08:50:29.587 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:50:28 2024 2024-09-14T08:50:29.587 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:50:29.587 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:50:29.588 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: baaf07f1-8725-4ffd-a1a5-efd3fb31c8b6 2024-09-14T08:50:29.588 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: true 2024-09-14T08:50:29.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n '' ']' 2024-09-14T08:50:29.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ stress_write_image cluster2 mirror image_33 2024-09-14T08:50:29.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:50:29.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:50:29.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_33 2024-09-14T08:50:29.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T08:50:29.592 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local duration=6 2024-09-14T08:50:29.592 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set +e 2024-09-14T08:50:29.592 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ timeout 6s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_33 --debug-rbd=20 --debug-journaler=20 2024-09-14T08:50:29.826 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:100 IOs, 47001600 bytes 2024-09-14T08:50:29.955 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:200 IOs, 68753408 bytes 2024-09-14T08:50:30.173 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:300 IOs, 128773120 bytes 2024-09-14T08:50:30.346 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:400 IOs, 168802304 bytes 2024-09-14T08:50:30.530 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:500 IOs, 210631680 bytes 2024-09-14T08:50:30.660 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:600 IOs, 232817664 bytes 2024-09-14T08:50:30.804 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:700 IOs, 255533056 bytes 2024-09-14T08:50:30.885 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:800 IOs, 280551424 bytes 2024-09-14T08:50:31.128 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:900 IOs, 338785280 bytes 2024-09-14T08:50:31.373 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1000 IOs, 382612480 bytes 2024-09-14T08:50:31.517 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1100 IOs, 410419200 bytes 2024-09-14T08:50:31.657 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1200 IOs, 452381696 bytes 2024-09-14T08:50:31.928 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1300 IOs, 513907712 bytes 2024-09-14T08:50:32.047 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1400 IOs, 531580928 bytes 2024-09-14T08:50:32.204 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1500 IOs, 570839040 bytes 2024-09-14T08:50:32.393 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1600 IOs, 607856640 bytes 2024-09-14T08:50:32.480 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1700 IOs, 608489472 bytes 2024-09-14T08:50:32.683 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1800 IOs, 662739968 bytes 2024-09-14T08:50:32.913 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1900 IOs, 722429952 bytes 2024-09-14T08:50:33.036 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2000 IOs, 736871424 bytes 2024-09-14T08:50:33.192 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2100 IOs, 767140864 bytes 2024-09-14T08:50:33.333 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2200 IOs, 792154112 bytes 2024-09-14T08:50:33.722 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2300 IOs, 919628800 bytes 2024-09-14T08:50:33.928 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2400 IOs, 968849408 bytes 2024-09-14T08:50:34.111 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2500 IOs, 1021950976 bytes 2024-09-14T08:50:34.371 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2600 IOs, 1088220160 bytes 2024-09-14T08:50:34.510 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2700 IOs, 1111231488 bytes 2024-09-14T08:50:34.677 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2800 IOs, 1151618048 bytes 2024-09-14T08:50:35.007 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2900 IOs, 1257726976 bytes 2024-09-14T08:50:35.165 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3000 IOs, 1280796672 bytes 2024-09-14T08:50:35.426 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3100 IOs, 1357632512 bytes 2024-09-14T08:50:35.598 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ error_code=124 2024-09-14T08:50:35.598 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set -e 2024-09-14T08:50:35.599 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T08:50:35.599 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:50:35.599 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T08:50:35.599 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_34 2024-09-14T08:50:35.599 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image_and_enable_mirror cluster2 mirror image_34 snapshot 128M 2024-09-14T08:50:35.599 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:50:35.599 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:50:35.599 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:50:35.599 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:50:35.599 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_34 2024-09-14T08:50:35.599 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:50:35.599 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:50:35.600 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n snapshot ']' 2024-09-14T08:50:35.600 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:50:35.600 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image cluster2 mirror image_34 128M 2024-09-14T08:50:35.600 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:50:35.600 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:50:35.600 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:50:35.600 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:50:35.600 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_34 2024-09-14T08:50:35.600 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:50:35.600 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local size=128 2024-09-14T08:50:35.600 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n 128M ']' 2024-09-14T08:50:35.600 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ size=128M 2024-09-14T08:50:35.600 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:50:35.600 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_34 2024-09-14T08:50:35.650 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T08:50:35.650 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:50:35.650 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' image = image ']' 2024-09-14T08:50:35.650 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:50:35.650 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ enable_mirror cluster2 mirror image_34 snapshot 2024-09-14T08:50:35.650 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:50:35.650 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:50:35.650 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_34 2024-09-14T08:50:35.651 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:50:35.651 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_34 snapshot 2024-09-14T08:50:36.541 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Mirroring enabled 2024-09-14T08:50:36.547 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 info mirror/image_34 2024-09-14T08:50:36.588 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_34': 2024-09-14T08:50:36.588 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T08:50:36.588 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:50:36.588 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 1 2024-09-14T08:50:36.588 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 141012c4a62d 2024-09-14T08:50:36.589 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.141012c4a62d 2024-09-14T08:50:36.589 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:50:36.589 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering 2024-09-14T08:50:36.589 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:50:36.589 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:50:36.589 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:50:35 2024 2024-09-14T08:50:36.589 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:50:35 2024 2024-09-14T08:50:36.589 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:50:35 2024 2024-09-14T08:50:36.589 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:50:36.589 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:50:36.589 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: ec5aa9c3-0e81-40f0-a328-f1ef0300304d 2024-09-14T08:50:36.589 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: true 2024-09-14T08:50:36.592 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n '' ']' 2024-09-14T08:50:36.592 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ stress_write_image cluster2 mirror image_34 2024-09-14T08:50:36.592 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:50:36.593 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:50:36.593 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_34 2024-09-14T08:50:36.593 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T08:50:36.594 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local duration=11 2024-09-14T08:50:36.594 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set +e 2024-09-14T08:50:36.594 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ timeout 11s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_34 --debug-rbd=20 --debug-journaler=20 2024-09-14T08:50:36.758 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:100 IOs, 20334592 bytes 2024-09-14T08:50:36.982 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:200 IOs, 74676224 bytes 2024-09-14T08:50:37.063 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:300 IOs, 79448064 bytes 2024-09-14T08:50:37.260 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:400 IOs, 121188352 bytes 2024-09-14T08:50:37.401 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:500 IOs, 161050624 bytes 2024-09-14T08:50:37.599 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:600 IOs, 189809664 bytes 2024-09-14T08:50:37.720 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:700 IOs, 210597888 bytes 2024-09-14T08:50:37.829 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:800 IOs, 221462528 bytes 2024-09-14T08:50:38.028 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:900 IOs, 273742848 bytes 2024-09-14T08:50:38.145 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1000 IOs, 289868800 bytes 2024-09-14T08:50:38.328 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1100 IOs, 331940864 bytes 2024-09-14T08:50:38.493 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1200 IOs, 373326848 bytes 2024-09-14T08:50:38.703 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1300 IOs, 427914240 bytes 2024-09-14T08:50:38.870 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1400 IOs, 458808320 bytes 2024-09-14T08:50:39.179 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1500 IOs, 541982720 bytes 2024-09-14T08:50:39.279 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1600 IOs, 549891072 bytes 2024-09-14T08:50:39.460 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1700 IOs, 591606784 bytes 2024-09-14T08:50:39.697 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1800 IOs, 665867264 bytes 2024-09-14T08:50:39.885 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1900 IOs, 702300160 bytes 2024-09-14T08:50:40.169 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2000 IOs, 790786048 bytes 2024-09-14T08:50:40.396 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2100 IOs, 836093952 bytes 2024-09-14T08:50:40.491 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2200 IOs, 836728832 bytes 2024-09-14T08:50:40.571 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2300 IOs, 838734848 bytes 2024-09-14T08:50:40.734 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2400 IOs, 870329344 bytes 2024-09-14T08:50:40.818 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2500 IOs, 873815040 bytes 2024-09-14T08:50:41.009 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2600 IOs, 927997952 bytes 2024-09-14T08:50:41.179 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2700 IOs, 960485376 bytes 2024-09-14T08:50:41.338 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2800 IOs, 982936576 bytes 2024-09-14T08:50:41.541 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2900 IOs, 1028350976 bytes 2024-09-14T08:50:41.716 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3000 IOs, 1068886016 bytes 2024-09-14T08:50:41.952 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3100 IOs, 1161088000 bytes 2024-09-14T08:50:42.315 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3200 IOs, 1240594432 bytes 2024-09-14T08:50:42.452 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3300 IOs, 1263730688 bytes 2024-09-14T08:50:42.536 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3400 IOs, 1287720960 bytes 2024-09-14T08:50:42.691 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3500 IOs, 1294017536 bytes 2024-09-14T08:50:42.904 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3600 IOs, 1338845184 bytes 2024-09-14T08:50:43.147 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3700 IOs, 1376678912 bytes 2024-09-14T08:50:43.273 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3800 IOs, 1403275264 bytes 2024-09-14T08:50:43.438 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3900 IOs, 1434371072 bytes 2024-09-14T08:50:43.624 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4000 IOs, 1477324800 bytes 2024-09-14T08:50:43.720 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4100 IOs, 1479527424 bytes 2024-09-14T08:50:43.807 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4200 IOs, 1503186944 bytes 2024-09-14T08:50:43.970 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4300 IOs, 1512590336 bytes 2024-09-14T08:50:44.192 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4400 IOs, 1572362240 bytes 2024-09-14T08:50:44.403 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4500 IOs, 1622947840 bytes 2024-09-14T08:50:44.493 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4600 IOs, 1628731392 bytes 2024-09-14T08:50:44.674 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4700 IOs, 1669370880 bytes 2024-09-14T08:50:44.826 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4800 IOs, 1699757056 bytes 2024-09-14T08:50:45.074 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4900 IOs, 1769329664 bytes 2024-09-14T08:50:45.294 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5000 IOs, 1821792256 bytes 2024-09-14T08:50:45.584 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5100 IOs, 1904754688 bytes 2024-09-14T08:50:45.697 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5200 IOs, 1920217088 bytes 2024-09-14T08:50:45.852 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5300 IOs, 1945635840 bytes 2024-09-14T08:50:46.076 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5400 IOs, 2001964032 bytes 2024-09-14T08:50:46.264 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5500 IOs, 2049274880 bytes 2024-09-14T08:50:46.516 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5600 IOs, 2116189184 bytes 2024-09-14T08:50:46.662 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5700 IOs, 2137408512 bytes 2024-09-14T08:50:46.880 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5800 IOs, 2194902016 bytes 2024-09-14T08:50:47.047 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5900 IOs, 2232401920 bytes 2024-09-14T08:50:47.275 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6000 IOs, 2286606336 bytes 2024-09-14T08:50:47.428 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6100 IOs, 2312976384 bytes 2024-09-14T08:50:47.600 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ error_code=124 2024-09-14T08:50:47.600 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set -e 2024-09-14T08:50:47.600 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T08:50:47.600 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:50:47.600 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T08:50:47.601 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_35 2024-09-14T08:50:47.601 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image_and_enable_mirror cluster2 mirror image_35 snapshot 128M 2024-09-14T08:50:47.601 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:50:47.601 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:50:47.601 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:50:47.601 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:50:47.601 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_35 2024-09-14T08:50:47.601 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:50:47.601 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:50:47.601 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n snapshot ']' 2024-09-14T08:50:47.601 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:50:47.601 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image cluster2 mirror image_35 128M 2024-09-14T08:50:47.601 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:50:47.601 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:50:47.602 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:50:47.602 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:50:47.602 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_35 2024-09-14T08:50:47.602 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:50:47.602 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local size=128 2024-09-14T08:50:47.602 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n 128M ']' 2024-09-14T08:50:47.602 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ size=128M 2024-09-14T08:50:47.602 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:50:47.602 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_35 2024-09-14T08:50:47.655 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T08:50:47.655 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:50:47.656 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' image = image ']' 2024-09-14T08:50:47.656 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:50:47.656 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ enable_mirror cluster2 mirror image_35 snapshot 2024-09-14T08:50:47.656 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:50:47.656 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:50:47.656 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_35 2024-09-14T08:50:47.656 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:50:47.656 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_35 snapshot 2024-09-14T08:50:48.582 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Mirroring enabled 2024-09-14T08:50:48.588 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 info mirror/image_35 2024-09-14T08:50:48.629 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_35': 2024-09-14T08:50:48.629 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T08:50:48.629 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:50:48.629 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 1 2024-09-14T08:50:48.629 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 141ca3136383 2024-09-14T08:50:48.629 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.141ca3136383 2024-09-14T08:50:48.629 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:50:48.630 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering 2024-09-14T08:50:48.630 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:50:48.630 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:50:48.630 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:50:47 2024 2024-09-14T08:50:48.630 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:50:47 2024 2024-09-14T08:50:48.630 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:50:47 2024 2024-09-14T08:50:48.630 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:50:48.630 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:50:48.630 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: 285cc24e-11c6-46a3-8120-f1a409abec99 2024-09-14T08:50:48.630 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: true 2024-09-14T08:50:48.633 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n '' ']' 2024-09-14T08:50:48.633 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ stress_write_image cluster2 mirror image_35 2024-09-14T08:50:48.633 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:50:48.633 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:50:48.633 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_35 2024-09-14T08:50:48.633 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T08:50:48.635 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local duration=10 2024-09-14T08:50:48.635 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set +e 2024-09-14T08:50:48.635 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ timeout 10s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_35 --debug-rbd=20 --debug-journaler=20 2024-09-14T08:50:49.011 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:100 IOs, 115664896 bytes 2024-09-14T08:50:49.176 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:200 IOs, 143030272 bytes 2024-09-14T08:50:49.354 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:300 IOs, 186690560 bytes 2024-09-14T08:50:49.530 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:400 IOs, 227098624 bytes 2024-09-14T08:50:49.704 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:500 IOs, 263865344 bytes 2024-09-14T08:50:49.778 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:600 IOs, 264489984 bytes 2024-09-14T08:50:49.885 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:700 IOs, 278244352 bytes 2024-09-14T08:50:50.051 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:800 IOs, 315142144 bytes 2024-09-14T08:50:50.244 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:900 IOs, 374912000 bytes 2024-09-14T08:50:50.558 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1000 IOs, 460186624 bytes 2024-09-14T08:50:50.781 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1100 IOs, 517194752 bytes 2024-09-14T08:50:50.867 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1200 IOs, 528693248 bytes 2024-09-14T08:50:51.026 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1300 IOs, 553444352 bytes 2024-09-14T08:50:51.208 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1400 IOs, 589623296 bytes 2024-09-14T08:50:51.376 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1500 IOs, 637542400 bytes 2024-09-14T08:50:51.599 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1600 IOs, 680437760 bytes 2024-09-14T08:50:51.731 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1700 IOs, 700863488 bytes 2024-09-14T08:50:51.986 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1800 IOs, 767681536 bytes 2024-09-14T08:50:52.200 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1900 IOs, 823022592 bytes 2024-09-14T08:50:52.341 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2000 IOs, 849935360 bytes 2024-09-14T08:50:52.534 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2100 IOs, 899241984 bytes 2024-09-14T08:50:52.830 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2200 IOs, 975339520 bytes 2024-09-14T08:50:53.013 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2300 IOs, 1024496640 bytes 2024-09-14T08:50:53.119 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2400 IOs, 1049982976 bytes 2024-09-14T08:50:53.341 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2500 IOs, 1089202176 bytes 2024-09-14T08:50:53.447 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2600 IOs, 1110850560 bytes 2024-09-14T08:50:53.662 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2700 IOs, 1161270272 bytes 2024-09-14T08:50:53.809 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2800 IOs, 1182935040 bytes 2024-09-14T08:50:53.954 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2900 IOs, 1210357760 bytes 2024-09-14T08:50:54.043 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3000 IOs, 1217558528 bytes 2024-09-14T08:50:54.243 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3100 IOs, 1264283648 bytes 2024-09-14T08:50:54.441 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3200 IOs, 1315088384 bytes 2024-09-14T08:50:54.534 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3300 IOs, 1326897152 bytes 2024-09-14T08:50:54.740 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3400 IOs, 1375416320 bytes 2024-09-14T08:50:54.880 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3500 IOs, 1401499648 bytes 2024-09-14T08:50:54.999 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3600 IOs, 1435000832 bytes 2024-09-14T08:50:55.195 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3700 IOs, 1467677696 bytes 2024-09-14T08:50:55.305 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3800 IOs, 1481139200 bytes 2024-09-14T08:50:55.461 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3900 IOs, 1513691136 bytes 2024-09-14T08:50:55.684 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4000 IOs, 1569475584 bytes 2024-09-14T08:50:55.906 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4100 IOs, 1636856832 bytes 2024-09-14T08:50:56.097 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4200 IOs, 1678095360 bytes 2024-09-14T08:50:56.221 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4300 IOs, 1698409472 bytes 2024-09-14T08:50:56.514 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4400 IOs, 1785465856 bytes 2024-09-14T08:50:56.685 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4500 IOs, 1821097984 bytes 2024-09-14T08:50:56.956 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4600 IOs, 1905848320 bytes 2024-09-14T08:50:57.068 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4700 IOs, 1925655552 bytes 2024-09-14T08:50:57.234 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4800 IOs, 1950384128 bytes 2024-09-14T08:50:57.459 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4900 IOs, 2013083648 bytes 2024-09-14T08:50:57.650 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5000 IOs, 2055791616 bytes 2024-09-14T08:50:57.825 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5100 IOs, 2098469888 bytes 2024-09-14T08:50:57.906 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5200 IOs, 2099088384 bytes 2024-09-14T08:50:57.984 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5300 IOs, 2113052672 bytes 2024-09-14T08:50:58.263 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5400 IOs, 2165125120 bytes 2024-09-14T08:50:58.411 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5500 IOs, 2192438272 bytes 2024-09-14T08:50:58.568 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5600 IOs, 2227356672 bytes 2024-09-14T08:50:58.640 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ error_code=124 2024-09-14T08:50:58.640 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set -e 2024-09-14T08:50:58.640 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T08:50:58.640 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:50:58.640 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T08:50:58.641 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_36 2024-09-14T08:50:58.641 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image_and_enable_mirror cluster2 mirror image_36 snapshot 128M 2024-09-14T08:50:58.641 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:50:58.641 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:50:58.641 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:50:58.641 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:50:58.641 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_36 2024-09-14T08:50:58.641 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:50:58.641 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:50:58.641 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n snapshot ']' 2024-09-14T08:50:58.641 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:50:58.641 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image cluster2 mirror image_36 128M 2024-09-14T08:50:58.641 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:50:58.642 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:50:58.642 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:50:58.642 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:50:58.642 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_36 2024-09-14T08:50:58.642 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:50:58.642 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local size=128 2024-09-14T08:50:58.642 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n 128M ']' 2024-09-14T08:50:58.642 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ size=128M 2024-09-14T08:50:58.642 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:50:58.642 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_36 2024-09-14T08:50:58.690 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T08:50:58.690 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:50:58.690 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' image = image ']' 2024-09-14T08:50:58.690 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:50:58.690 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ enable_mirror cluster2 mirror image_36 snapshot 2024-09-14T08:50:58.690 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:50:58.690 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:50:58.691 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_36 2024-09-14T08:50:58.691 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:50:58.691 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_36 snapshot 2024-09-14T08:50:59.605 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Mirroring enabled 2024-09-14T08:50:59.610 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 info mirror/image_36 2024-09-14T08:50:59.652 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_36': 2024-09-14T08:50:59.652 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T08:50:59.653 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:50:59.653 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 1 2024-09-14T08:50:59.653 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 1428921a5b93 2024-09-14T08:50:59.653 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.1428921a5b93 2024-09-14T08:50:59.653 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:50:59.653 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering 2024-09-14T08:50:59.653 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:50:59.653 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:50:59.653 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:50:58 2024 2024-09-14T08:50:59.653 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:50:58 2024 2024-09-14T08:50:59.653 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:50:58 2024 2024-09-14T08:50:59.653 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:50:59.653 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:50:59.653 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: 1e059464-8982-44b9-a52b-107ceeccb93e 2024-09-14T08:50:59.654 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: true 2024-09-14T08:50:59.654 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n '' ']' 2024-09-14T08:50:59.654 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ stress_write_image cluster2 mirror image_36 2024-09-14T08:50:59.654 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:50:59.654 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:50:59.655 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_36 2024-09-14T08:50:59.655 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T08:50:59.655 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local duration=8 2024-09-14T08:50:59.655 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set +e 2024-09-14T08:50:59.655 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ timeout 8s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_36 --debug-rbd=20 --debug-journaler=20 2024-09-14T08:50:59.914 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:100 IOs, 58952704 bytes 2024-09-14T08:51:00.112 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:200 IOs, 129736704 bytes 2024-09-14T08:51:00.376 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:300 IOs, 184236032 bytes 2024-09-14T08:51:00.669 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:400 IOs, 270396416 bytes 2024-09-14T08:51:00.934 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:500 IOs, 347437056 bytes 2024-09-14T08:51:01.284 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:600 IOs, 456739840 bytes 2024-09-14T08:51:01.460 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:700 IOs, 502659072 bytes 2024-09-14T08:51:01.659 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:800 IOs, 543049728 bytes 2024-09-14T08:51:01.784 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:900 IOs, 569956352 bytes 2024-09-14T08:51:02.081 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1000 IOs, 662021120 bytes 2024-09-14T08:51:02.256 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1100 IOs, 686729216 bytes 2024-09-14T08:51:02.378 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1200 IOs, 710228992 bytes 2024-09-14T08:51:02.545 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1300 IOs, 748402688 bytes 2024-09-14T08:51:02.713 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1400 IOs, 772050944 bytes 2024-09-14T08:51:02.840 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1500 IOs, 808452096 bytes 2024-09-14T08:51:03.041 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1600 IOs, 855852032 bytes 2024-09-14T08:51:03.284 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1700 IOs, 917509120 bytes 2024-09-14T08:51:03.570 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1800 IOs, 981818368 bytes 2024-09-14T08:51:03.778 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1900 IOs, 1032264704 bytes 2024-09-14T08:51:03.988 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2000 IOs, 1097549824 bytes 2024-09-14T08:51:04.172 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2100 IOs, 1126410240 bytes 2024-09-14T08:51:04.314 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2200 IOs, 1171982336 bytes 2024-09-14T08:51:04.557 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2300 IOs, 1223250944 bytes 2024-09-14T08:51:04.843 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2400 IOs, 1320845312 bytes 2024-09-14T08:51:04.985 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2500 IOs, 1328741376 bytes 2024-09-14T08:51:05.129 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2600 IOs, 1352198144 bytes 2024-09-14T08:51:05.285 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2700 IOs, 1383248896 bytes 2024-09-14T08:51:05.537 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2800 IOs, 1466131456 bytes 2024-09-14T08:51:05.808 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2900 IOs, 1534418944 bytes 2024-09-14T08:51:05.926 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3000 IOs, 1548761088 bytes 2024-09-14T08:51:06.123 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3100 IOs, 1600707584 bytes 2024-09-14T08:51:06.278 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3200 IOs, 1624916992 bytes 2024-09-14T08:51:06.443 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3300 IOs, 1659106304 bytes 2024-09-14T08:51:06.638 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3400 IOs, 1701641216 bytes 2024-09-14T08:51:06.788 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3500 IOs, 1734657024 bytes 2024-09-14T08:51:07.046 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3600 IOs, 1806924800 bytes 2024-09-14T08:51:07.226 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3700 IOs, 1839736832 bytes 2024-09-14T08:51:07.407 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3800 IOs, 1876426752 bytes 2024-09-14T08:51:07.550 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3900 IOs, 1903950848 bytes 2024-09-14T08:51:07.663 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ error_code=124 2024-09-14T08:51:07.663 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set -e 2024-09-14T08:51:07.663 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T08:51:07.663 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:51:07.663 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T08:51:07.663 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_37 2024-09-14T08:51:07.663 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image_and_enable_mirror cluster2 mirror image_37 snapshot 128M 2024-09-14T08:51:07.663 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:51:07.664 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:51:07.664 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:51:07.664 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:51:07.664 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_37 2024-09-14T08:51:07.664 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:51:07.664 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:51:07.664 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n snapshot ']' 2024-09-14T08:51:07.664 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:51:07.664 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image cluster2 mirror image_37 128M 2024-09-14T08:51:07.664 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:51:07.664 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:51:07.664 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:51:07.664 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:51:07.664 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_37 2024-09-14T08:51:07.665 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:51:07.665 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local size=128 2024-09-14T08:51:07.665 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n 128M ']' 2024-09-14T08:51:07.665 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ size=128M 2024-09-14T08:51:07.665 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:51:07.665 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_37 2024-09-14T08:51:07.717 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T08:51:07.717 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:51:07.717 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' image = image ']' 2024-09-14T08:51:07.717 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:51:07.717 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ enable_mirror cluster2 mirror image_37 snapshot 2024-09-14T08:51:07.717 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:51:07.717 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:51:07.717 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_37 2024-09-14T08:51:07.717 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:51:07.717 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_37 snapshot 2024-09-14T08:51:08.615 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Mirroring enabled 2024-09-14T08:51:08.621 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 info mirror/image_37 2024-09-14T08:51:08.662 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_37': 2024-09-14T08:51:08.663 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T08:51:08.663 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:51:08.663 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 1 2024-09-14T08:51:08.663 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 1434418e16fa 2024-09-14T08:51:08.663 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.1434418e16fa 2024-09-14T08:51:08.663 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:51:08.663 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering 2024-09-14T08:51:08.663 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:51:08.663 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:51:08.663 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:51:07 2024 2024-09-14T08:51:08.663 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:51:07 2024 2024-09-14T08:51:08.663 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:51:07 2024 2024-09-14T08:51:08.663 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:51:08.663 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:51:08.664 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: f4bb57da-c08d-42af-8cd0-9868b7375dfb 2024-09-14T08:51:08.664 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: true 2024-09-14T08:51:08.666 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n '' ']' 2024-09-14T08:51:08.666 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ stress_write_image cluster2 mirror image_37 2024-09-14T08:51:08.666 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:51:08.666 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:51:08.667 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_37 2024-09-14T08:51:08.667 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T08:51:08.668 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local duration=6 2024-09-14T08:51:08.668 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set +e 2024-09-14T08:51:08.668 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ timeout 6s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_37 --debug-rbd=20 --debug-journaler=20 2024-09-14T08:51:08.894 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:100 IOs, 40939520 bytes 2024-09-14T08:51:09.018 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:200 IOs, 58912768 bytes 2024-09-14T08:51:09.297 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:300 IOs, 132894720 bytes 2024-09-14T08:51:09.412 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:400 IOs, 164436992 bytes 2024-09-14T08:51:09.534 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:500 IOs, 165058560 bytes 2024-09-14T08:51:09.727 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:600 IOs, 207772672 bytes 2024-09-14T08:51:09.845 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:700 IOs, 224930816 bytes 2024-09-14T08:51:10.121 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:800 IOs, 298312704 bytes 2024-09-14T08:51:10.204 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:900 IOs, 298933248 bytes 2024-09-14T08:51:10.282 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1000 IOs, 299534336 bytes 2024-09-14T08:51:10.389 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1100 IOs, 311304192 bytes 2024-09-14T08:51:10.469 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1200 IOs, 311913472 bytes 2024-09-14T08:51:10.608 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1300 IOs, 336310272 bytes 2024-09-14T08:51:10.694 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1400 IOs, 338031616 bytes 2024-09-14T08:51:10.829 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1500 IOs, 362983424 bytes 2024-09-14T08:51:10.987 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1600 IOs, 390805504 bytes 2024-09-14T08:51:11.138 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1700 IOs, 417220608 bytes 2024-09-14T08:51:11.399 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1800 IOs, 489054208 bytes 2024-09-14T08:51:11.640 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1900 IOs, 552970240 bytes 2024-09-14T08:51:11.774 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2000 IOs, 581779456 bytes 2024-09-14T08:51:11.985 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2100 IOs, 631535616 bytes 2024-09-14T08:51:12.220 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2200 IOs, 679900160 bytes 2024-09-14T08:51:12.300 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2300 IOs, 680503296 bytes 2024-09-14T08:51:12.499 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2400 IOs, 721553408 bytes 2024-09-14T08:51:12.661 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2500 IOs, 753412096 bytes 2024-09-14T08:51:12.901 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2600 IOs, 815578112 bytes 2024-09-14T08:51:13.113 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2700 IOs, 870825984 bytes 2024-09-14T08:51:13.214 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2800 IOs, 877842432 bytes 2024-09-14T08:51:13.374 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2900 IOs, 911668224 bytes 2024-09-14T08:51:13.478 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3000 IOs, 916422656 bytes 2024-09-14T08:51:13.562 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3100 IOs, 919522304 bytes 2024-09-14T08:51:13.776 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3200 IOs, 975753216 bytes 2024-09-14T08:51:14.173 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3300 IOs, 1100563456 bytes 2024-09-14T08:51:14.438 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3400 IOs, 1166909440 bytes 2024-09-14T08:51:14.675 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ error_code=124 2024-09-14T08:51:14.676 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set -e 2024-09-14T08:51:14.676 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T08:51:14.676 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:51:14.676 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T08:51:14.676 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_38 2024-09-14T08:51:14.676 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image_and_enable_mirror cluster2 mirror image_38 snapshot 128M 2024-09-14T08:51:14.676 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:51:14.676 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:51:14.676 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:51:14.676 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:51:14.676 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_38 2024-09-14T08:51:14.676 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:51:14.677 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:51:14.677 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n snapshot ']' 2024-09-14T08:51:14.677 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:51:14.677 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image cluster2 mirror image_38 128M 2024-09-14T08:51:14.677 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:51:14.677 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:51:14.677 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:51:14.677 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:51:14.677 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_38 2024-09-14T08:51:14.677 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:51:14.677 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local size=128 2024-09-14T08:51:14.677 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n 128M ']' 2024-09-14T08:51:14.677 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ size=128M 2024-09-14T08:51:14.678 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:51:14.678 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_38 2024-09-14T08:51:14.725 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T08:51:14.725 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:51:14.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' image = image ']' 2024-09-14T08:51:14.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:51:14.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ enable_mirror cluster2 mirror image_38 snapshot 2024-09-14T08:51:14.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:51:14.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:51:14.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_38 2024-09-14T08:51:14.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:51:14.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_38 snapshot 2024-09-14T08:51:15.147 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Mirroring enabled 2024-09-14T08:51:15.153 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 info mirror/image_38 2024-09-14T08:51:15.192 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_38': 2024-09-14T08:51:15.192 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T08:51:15.192 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:51:15.193 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 1 2024-09-14T08:51:15.193 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 14409cbb3992 2024-09-14T08:51:15.193 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.14409cbb3992 2024-09-14T08:51:15.193 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:51:15.193 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering 2024-09-14T08:51:15.193 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:51:15.193 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:51:15.193 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:51:14 2024 2024-09-14T08:51:15.193 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:51:14 2024 2024-09-14T08:51:15.193 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:51:14 2024 2024-09-14T08:51:15.193 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:51:15.193 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:51:15.194 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: 56c69995-3ca1-4975-bc59-f14d91322ad4 2024-09-14T08:51:15.194 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: true 2024-09-14T08:51:15.196 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n '' ']' 2024-09-14T08:51:15.196 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ stress_write_image cluster2 mirror image_38 2024-09-14T08:51:15.196 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:51:15.196 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:51:15.196 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_38 2024-09-14T08:51:15.196 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T08:51:15.198 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local duration=9 2024-09-14T08:51:15.198 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set +e 2024-09-14T08:51:15.198 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ timeout 9s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_38 --debug-rbd=20 --debug-journaler=20 2024-09-14T08:51:15.507 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:100 IOs, 91364352 bytes 2024-09-14T08:51:15.706 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:200 IOs, 123845632 bytes 2024-09-14T08:51:15.865 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:300 IOs, 157946880 bytes 2024-09-14T08:51:16.084 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:400 IOs, 215685120 bytes 2024-09-14T08:51:16.205 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:500 IOs, 233648128 bytes 2024-09-14T08:51:16.412 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:600 IOs, 290741248 bytes 2024-09-14T08:51:16.685 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:700 IOs, 361159680 bytes 2024-09-14T08:51:16.816 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:800 IOs, 381631488 bytes 2024-09-14T08:51:16.957 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:900 IOs, 407560192 bytes 2024-09-14T08:51:17.058 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1000 IOs, 417281024 bytes 2024-09-14T08:51:17.199 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1100 IOs, 445338624 bytes 2024-09-14T08:51:17.356 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1200 IOs, 474649600 bytes 2024-09-14T08:51:17.561 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1300 IOs, 529227776 bytes 2024-09-14T08:51:17.702 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1400 IOs, 550356992 bytes 2024-09-14T08:51:17.865 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1500 IOs, 584233984 bytes 2024-09-14T08:51:18.009 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1600 IOs, 623873024 bytes 2024-09-14T08:51:18.286 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1700 IOs, 687907840 bytes 2024-09-14T08:51:18.513 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1800 IOs, 745416704 bytes 2024-09-14T08:51:18.589 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1900 IOs, 746034176 bytes 2024-09-14T08:51:18.669 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2000 IOs, 746644480 bytes 2024-09-14T08:51:18.841 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2100 IOs, 784512000 bytes 2024-09-14T08:51:18.920 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2200 IOs, 785132544 bytes 2024-09-14T08:51:19.114 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2300 IOs, 840352768 bytes 2024-09-14T08:51:19.429 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2400 IOs, 927108096 bytes 2024-09-14T08:51:19.549 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2500 IOs, 942307328 bytes 2024-09-14T08:51:19.667 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2600 IOs, 959592448 bytes 2024-09-14T08:51:19.773 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2700 IOs, 971631616 bytes 2024-09-14T08:51:19.973 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2800 IOs, 1024660480 bytes 2024-09-14T08:51:20.134 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2900 IOs, 1053034496 bytes 2024-09-14T08:51:20.315 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3000 IOs, 1094181888 bytes 2024-09-14T08:51:20.478 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3100 IOs, 1123534848 bytes 2024-09-14T08:51:20.690 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3200 IOs, 1178218496 bytes 2024-09-14T08:51:20.859 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3300 IOs, 1211936768 bytes 2024-09-14T08:51:20.998 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3400 IOs, 1234762752 bytes 2024-09-14T08:51:21.221 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3500 IOs, 1291995136 bytes 2024-09-14T08:51:21.401 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3600 IOs, 1326178304 bytes 2024-09-14T08:51:21.661 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3700 IOs, 1401272320 bytes 2024-09-14T08:51:21.907 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3800 IOs, 1473419264 bytes 2024-09-14T08:51:22.088 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3900 IOs, 1510677504 bytes 2024-09-14T08:51:22.226 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4000 IOs, 1537709056 bytes 2024-09-14T08:51:22.533 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4100 IOs, 1623595008 bytes 2024-09-14T08:51:22.766 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4200 IOs, 1684257792 bytes 2024-09-14T08:51:22.952 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4300 IOs, 1729895424 bytes 2024-09-14T08:51:23.099 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4400 IOs, 1755976704 bytes 2024-09-14T08:51:23.299 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4500 IOs, 1809447936 bytes 2024-09-14T08:51:23.553 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4600 IOs, 1842876416 bytes 2024-09-14T08:51:23.733 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4700 IOs, 1887903744 bytes 2024-09-14T08:51:23.945 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4800 IOs, 1935177728 bytes 2024-09-14T08:51:24.086 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4900 IOs, 1961265152 bytes 2024-09-14T08:51:24.203 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ error_code=124 2024-09-14T08:51:24.203 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set -e 2024-09-14T08:51:24.203 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T08:51:24.203 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:51:24.203 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T08:51:24.203 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_39 2024-09-14T08:51:24.203 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image_and_enable_mirror cluster2 mirror image_39 snapshot 128M 2024-09-14T08:51:24.203 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:51:24.203 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:51:24.204 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:51:24.204 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:51:24.204 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_39 2024-09-14T08:51:24.204 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:51:24.204 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:51:24.204 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n snapshot ']' 2024-09-14T08:51:24.204 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:51:24.204 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image cluster2 mirror image_39 128M 2024-09-14T08:51:24.204 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:51:24.204 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:51:24.204 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:51:24.204 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:51:24.204 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_39 2024-09-14T08:51:24.204 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:51:24.205 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local size=128 2024-09-14T08:51:24.205 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n 128M ']' 2024-09-14T08:51:24.205 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ size=128M 2024-09-14T08:51:24.205 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:51:24.205 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_39 2024-09-14T08:51:24.253 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T08:51:24.253 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:51:24.253 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' image = image ']' 2024-09-14T08:51:24.253 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:51:24.253 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ enable_mirror cluster2 mirror image_39 snapshot 2024-09-14T08:51:24.253 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:51:24.253 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:51:24.253 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_39 2024-09-14T08:51:24.253 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:51:24.253 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_39 snapshot 2024-09-14T08:51:25.163 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Mirroring enabled 2024-09-14T08:51:25.169 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 info mirror/image_39 2024-09-14T08:51:25.211 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_39': 2024-09-14T08:51:25.212 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T08:51:25.212 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:51:25.212 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 1 2024-09-14T08:51:25.212 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 144cc0542cc9 2024-09-14T08:51:25.212 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.144cc0542cc9 2024-09-14T08:51:25.212 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:51:25.212 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering 2024-09-14T08:51:25.212 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:51:25.212 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:51:25.212 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:51:24 2024 2024-09-14T08:51:25.212 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:51:24 2024 2024-09-14T08:51:25.212 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:51:24 2024 2024-09-14T08:51:25.212 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:51:25.212 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:51:25.213 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: e2dde19c-e7be-4645-8807-0ea1479cbd41 2024-09-14T08:51:25.213 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: true 2024-09-14T08:51:25.215 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n '' ']' 2024-09-14T08:51:25.215 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ stress_write_image cluster2 mirror image_39 2024-09-14T08:51:25.215 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:51:25.215 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:51:25.215 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_39 2024-09-14T08:51:25.215 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T08:51:25.217 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local duration=12 2024-09-14T08:51:25.217 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set +e 2024-09-14T08:51:25.217 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ timeout 12s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_39 --debug-rbd=20 --debug-journaler=20 2024-09-14T08:51:25.487 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:100 IOs, 60797952 bytes 2024-09-14T08:51:25.610 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:200 IOs, 78363648 bytes 2024-09-14T08:51:25.773 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:300 IOs, 115076096 bytes 2024-09-14T08:51:25.911 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:400 IOs, 137995264 bytes 2024-09-14T08:51:26.161 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:500 IOs, 229911552 bytes 2024-09-14T08:51:26.294 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:600 IOs, 237349888 bytes 2024-09-14T08:51:26.474 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:700 IOs, 280317952 bytes 2024-09-14T08:51:26.720 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:800 IOs, 341493760 bytes 2024-09-14T08:51:26.851 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:900 IOs, 363736064 bytes 2024-09-14T08:51:27.013 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1000 IOs, 397149184 bytes 2024-09-14T08:51:27.170 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1100 IOs, 426715136 bytes 2024-09-14T08:51:27.380 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1200 IOs, 474246144 bytes 2024-09-14T08:51:27.458 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1300 IOs, 475361280 bytes 2024-09-14T08:51:27.629 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1400 IOs, 515876864 bytes 2024-09-14T08:51:27.755 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1500 IOs, 535997440 bytes 2024-09-14T08:51:27.882 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1600 IOs, 558563328 bytes 2024-09-14T08:51:28.019 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1700 IOs, 586091520 bytes 2024-09-14T08:51:28.227 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1800 IOs, 631757824 bytes 2024-09-14T08:51:28.409 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1900 IOs, 673899520 bytes 2024-09-14T08:51:28.557 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2000 IOs, 704536576 bytes 2024-09-14T08:51:28.674 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2100 IOs, 726646784 bytes 2024-09-14T08:51:28.890 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2200 IOs, 772221952 bytes 2024-09-14T08:51:29.025 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2300 IOs, 795004928 bytes 2024-09-14T08:51:29.402 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2400 IOs, 915837952 bytes 2024-09-14T08:51:29.690 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2500 IOs, 1006729216 bytes 2024-09-14T08:51:29.780 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2600 IOs, 1011162112 bytes 2024-09-14T08:51:29.957 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2700 IOs, 1048278016 bytes 2024-09-14T08:51:30.172 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2800 IOs, 1108935680 bytes 2024-09-14T08:51:30.387 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2900 IOs, 1181155328 bytes 2024-09-14T08:51:30.659 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3000 IOs, 1248261120 bytes 2024-09-14T08:51:30.757 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3100 IOs, 1248846848 bytes 2024-09-14T08:51:31.036 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3200 IOs, 1334128640 bytes 2024-09-14T08:51:31.293 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3300 IOs, 1426246656 bytes 2024-09-14T08:51:31.540 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3400 IOs, 1480550400 bytes 2024-09-14T08:51:31.681 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3500 IOs, 1498109952 bytes 2024-09-14T08:51:31.933 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3600 IOs, 1569179648 bytes 2024-09-14T08:51:32.094 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3700 IOs, 1604866048 bytes 2024-09-14T08:51:32.190 INFO:tasks.ceph.cluster2.osd.0.smithi164.stderr:2024-09-14T08:51:32.190+0000 7f7ce7714640 -1 reset not still connected to 0x55a087aea750 2024-09-14T08:51:32.191 INFO:tasks.ceph.cluster2.osd.0.smithi164.stderr:2024-09-14T08:51:32.190+0000 7f7ce7714640 -1 reset not still connected to 0x55a08961ac30 2024-09-14T08:51:32.191 INFO:tasks.ceph.cluster2.osd.0.smithi164.stderr:2024-09-14T08:51:32.190+0000 7f7ce7714640 -1 reset not still connected to 0x55a08ada3520 2024-09-14T08:51:32.191 INFO:tasks.ceph.cluster2.osd.0.smithi164.stderr:2024-09-14T08:51:32.190+0000 7f7ce7714640 -1 reset not still connected to 0x55a08ada36c0 2024-09-14T08:51:32.436 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3800 IOs, 1740929024 bytes 2024-09-14T08:51:32.616 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3900 IOs, 1752598528 bytes 2024-09-14T08:51:32.750 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4000 IOs, 1774997504 bytes 2024-09-14T08:51:32.931 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4100 IOs, 1817922560 bytes 2024-09-14T08:51:33.111 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4200 IOs, 1860305920 bytes 2024-09-14T08:51:33.383 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4300 IOs, 1947151360 bytes 2024-09-14T08:51:33.529 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4400 IOs, 1973874688 bytes 2024-09-14T08:51:33.718 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4500 IOs, 2022008832 bytes 2024-09-14T08:51:33.864 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4600 IOs, 2052054016 bytes 2024-09-14T08:51:34.178 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4700 IOs, 2168058880 bytes 2024-09-14T08:51:34.375 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4800 IOs, 2193242112 bytes 2024-09-14T08:51:34.458 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4900 IOs, 2193881088 bytes 2024-09-14T08:51:34.623 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5000 IOs, 2230375424 bytes 2024-09-14T08:51:34.915 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5100 IOs, 2316966912 bytes 2024-09-14T08:51:35.096 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5200 IOs, 2357812224 bytes 2024-09-14T08:51:35.221 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5300 IOs, 2374337536 bytes 2024-09-14T08:51:35.438 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5400 IOs, 2430736384 bytes 2024-09-14T08:51:35.551 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5500 IOs, 2444378112 bytes 2024-09-14T08:51:35.725 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5600 IOs, 2485466112 bytes 2024-09-14T08:51:35.850 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5700 IOs, 2500851712 bytes 2024-09-14T08:51:36.022 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5800 IOs, 2542234624 bytes 2024-09-14T08:51:36.282 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5900 IOs, 2612696064 bytes 2024-09-14T08:51:36.471 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6000 IOs, 2678845440 bytes 2024-09-14T08:51:36.694 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6100 IOs, 2715628544 bytes 2024-09-14T08:51:36.849 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6200 IOs, 2744492032 bytes 2024-09-14T08:51:36.976 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6300 IOs, 2776359936 bytes 2024-09-14T08:51:37.223 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ error_code=124 2024-09-14T08:51:37.223 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set -e 2024-09-14T08:51:37.223 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T08:51:37.223 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:51:37.223 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T08:51:37.223 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_40 2024-09-14T08:51:37.223 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image_and_enable_mirror cluster2 mirror image_40 snapshot 128M 2024-09-14T08:51:37.223 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:51:37.223 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:51:37.223 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:51:37.223 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:51:37.224 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_40 2024-09-14T08:51:37.224 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:51:37.224 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:51:37.224 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n snapshot ']' 2024-09-14T08:51:37.224 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:51:37.224 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image cluster2 mirror image_40 128M 2024-09-14T08:51:37.224 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:51:37.224 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:51:37.224 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:51:37.224 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:51:37.224 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_40 2024-09-14T08:51:37.224 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:51:37.224 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local size=128 2024-09-14T08:51:37.225 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n 128M ']' 2024-09-14T08:51:37.225 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ size=128M 2024-09-14T08:51:37.225 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:51:37.225 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_40 2024-09-14T08:51:37.272 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T08:51:37.272 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:51:37.272 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' image = image ']' 2024-09-14T08:51:37.272 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:51:37.272 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ enable_mirror cluster2 mirror image_40 snapshot 2024-09-14T08:51:37.272 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:51:37.273 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:51:37.273 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_40 2024-09-14T08:51:37.273 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:51:37.273 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_40 snapshot 2024-09-14T08:51:38.183 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Mirroring enabled 2024-09-14T08:51:38.189 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 info mirror/image_40 2024-09-14T08:51:38.231 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_40': 2024-09-14T08:51:38.231 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T08:51:38.231 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:51:38.231 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 1 2024-09-14T08:51:38.232 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 1458fc96510f 2024-09-14T08:51:38.232 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.1458fc96510f 2024-09-14T08:51:38.232 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:51:38.232 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering 2024-09-14T08:51:38.232 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:51:38.232 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:51:38.232 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:51:37 2024 2024-09-14T08:51:38.232 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:51:37 2024 2024-09-14T08:51:38.232 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:51:37 2024 2024-09-14T08:51:38.232 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:51:38.232 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:51:38.232 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: 3327bc1e-5162-4a18-8a41-c4a1c978f5cf 2024-09-14T08:51:38.232 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: true 2024-09-14T08:51:38.235 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n '' ']' 2024-09-14T08:51:38.236 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ stress_write_image cluster2 mirror image_40 2024-09-14T08:51:38.236 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:51:38.236 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:51:38.236 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_40 2024-09-14T08:51:38.236 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T08:51:38.237 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local duration=13 2024-09-14T08:51:38.237 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set +e 2024-09-14T08:51:38.237 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ timeout 13s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_40 --debug-rbd=20 --debug-journaler=20 2024-09-14T08:51:38.414 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:100 IOs, 21550080 bytes 2024-09-14T08:51:38.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:200 IOs, 70117376 bytes 2024-09-14T08:51:38.823 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:300 IOs, 125599744 bytes 2024-09-14T08:51:38.974 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:400 IOs, 156421120 bytes 2024-09-14T08:51:39.071 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:500 IOs, 175419392 bytes 2024-09-14T08:51:39.291 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:600 IOs, 211926016 bytes 2024-09-14T08:51:39.432 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:700 IOs, 247151616 bytes 2024-09-14T08:51:39.636 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:800 IOs, 286748672 bytes 2024-09-14T08:51:39.838 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:900 IOs, 352186368 bytes 2024-09-14T08:51:40.006 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1000 IOs, 382222336 bytes 2024-09-14T08:51:40.128 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1100 IOs, 411668480 bytes 2024-09-14T08:51:40.432 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1200 IOs, 498376704 bytes 2024-09-14T08:51:40.645 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1300 IOs, 553171968 bytes 2024-09-14T08:51:40.847 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1400 IOs, 599425024 bytes 2024-09-14T08:51:41.115 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1500 IOs, 661227520 bytes 2024-09-14T08:51:41.348 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1600 IOs, 717534208 bytes 2024-09-14T08:51:41.562 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1700 IOs, 772526080 bytes 2024-09-14T08:51:41.806 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1800 IOs, 834087936 bytes 2024-09-14T08:51:41.936 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1900 IOs, 872260608 bytes 2024-09-14T08:51:42.237 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2000 IOs, 942717952 bytes 2024-09-14T08:51:42.403 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2100 IOs, 976731136 bytes 2024-09-14T08:51:42.633 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2200 IOs, 1043407872 bytes 2024-09-14T08:51:42.762 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2300 IOs, 1056650240 bytes 2024-09-14T08:51:43.035 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2400 IOs, 1133230080 bytes 2024-09-14T08:51:43.146 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.7990168469092966) > weight (0.25) 2024-09-14T08:51:43.146 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.27410611828911435) > weight (0.25) 2024-09-14T08:51:43.146 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.43603243543445214) > weight (0.25) 2024-09-14T08:51:43.146 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.3 with skip (0.44052347793392377) > weight (0.25) 2024-09-14T08:51:43.147 INFO:tasks.rbd_mirror_thrash:waiting for 64 secs before thrashing 2024-09-14T08:51:43.269 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2500 IOs, 1197614080 bytes 2024-09-14T08:51:43.447 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2600 IOs, 1235343360 bytes 2024-09-14T08:51:43.594 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2700 IOs, 1261070336 bytes 2024-09-14T08:51:43.801 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2800 IOs, 1313540096 bytes 2024-09-14T08:51:43.972 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2900 IOs, 1372715008 bytes 2024-09-14T08:51:44.243 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3000 IOs, 1427937280 bytes 2024-09-14T08:51:44.391 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3100 IOs, 1461207040 bytes 2024-09-14T08:51:44.634 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3200 IOs, 1524250624 bytes 2024-09-14T08:51:44.715 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3300 IOs, 1524862976 bytes 2024-09-14T08:51:44.844 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3400 IOs, 1544055808 bytes 2024-09-14T08:51:44.922 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3500 IOs, 1544691712 bytes 2024-09-14T08:51:45.139 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3600 IOs, 1598565376 bytes 2024-09-14T08:51:45.269 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3700 IOs, 1621582848 bytes 2024-09-14T08:51:45.456 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3800 IOs, 1663730688 bytes 2024-09-14T08:51:45.600 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3900 IOs, 1690741760 bytes 2024-09-14T08:51:45.784 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4000 IOs, 1752157184 bytes 2024-09-14T08:51:45.996 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4100 IOs, 1788001280 bytes 2024-09-14T08:51:46.157 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4200 IOs, 1821221888 bytes 2024-09-14T08:51:46.237 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4300 IOs, 1821851648 bytes 2024-09-14T08:51:46.367 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4400 IOs, 1854135296 bytes 2024-09-14T08:51:46.508 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4500 IOs, 1868510208 bytes 2024-09-14T08:51:46.601 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4600 IOs, 1869115392 bytes 2024-09-14T08:51:46.823 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4700 IOs, 1923946496 bytes 2024-09-14T08:51:46.924 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.7370965226968649) > weight (0.25) 2024-09-14T08:51:46.924 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.9056668685131444) > weight (0.25) 2024-09-14T08:51:46.924 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.8169248600584018) > weight (0.25) 2024-09-14T08:51:46.924 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.3 2024-09-14T08:51:46.925 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Sent signal 15 2024-09-14T08:51:46.925 INFO:tasks.rbd_mirror_thrash:waiting for 2 secs before reviving daemons 2024-09-14T08:51:46.926 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi164.stderr:2024-09-14T08:51:46.925+0000 7f1153646640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.3 (PID: 36919) UID: 1000 2024-09-14T08:51:47.076 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4800 IOs, 1987789824 bytes 2024-09-14T08:51:47.206 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4900 IOs, 2003709952 bytes 2024-09-14T08:51:47.311 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5000 IOs, 2021208064 bytes 2024-09-14T08:51:47.498 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5100 IOs, 2081586176 bytes 2024-09-14T08:51:47.732 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5200 IOs, 2131259392 bytes 2024-09-14T08:51:47.907 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5300 IOs, 2154716160 bytes 2024-09-14T08:51:48.047 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5400 IOs, 2179101696 bytes 2024-09-14T08:51:48.156 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5500 IOs, 2191364096 bytes 2024-09-14T08:51:48.277 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5600 IOs, 2209025024 bytes 2024-09-14T08:51:48.486 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5700 IOs, 2259937280 bytes 2024-09-14T08:51:48.640 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5800 IOs, 2293422080 bytes 2024-09-14T08:51:48.842 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5900 IOs, 2339398656 bytes 2024-09-14T08:51:48.924 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6000 IOs, 2341396480 bytes 2024-09-14T08:51:48.926 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.3 2024-09-14T08:51:48.926 INFO:teuthology.orchestra.run:waiting for 600 2024-09-14T08:51:48.927 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.3 2024-09-14T08:51:48.927 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Restarting daemon 2024-09-14T08:51:48.927 DEBUG:teuthology.orchestra.run.smithi164:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.3 2024-09-14T08:51:48.930 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Started 2024-09-14T08:51:48.930 INFO:tasks.rbd_mirror_thrash:waiting for 60 secs before thrashing 2024-09-14T08:51:49.014 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6100 IOs, 2342024192 bytes 2024-09-14T08:51:49.244 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6200 IOs, 2398559232 bytes 2024-09-14T08:51:49.413 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6300 IOs, 2426523648 bytes 2024-09-14T08:51:49.492 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6400 IOs, 2427143168 bytes 2024-09-14T08:51:49.663 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6500 IOs, 2463831040 bytes 2024-09-14T08:51:49.841 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6600 IOs, 2503843840 bytes 2024-09-14T08:51:50.065 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6700 IOs, 2562225152 bytes 2024-09-14T08:51:50.313 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6800 IOs, 2642782208 bytes 2024-09-14T08:51:50.555 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6900 IOs, 2701239296 bytes 2024-09-14T08:51:50.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:7000 IOs, 2729175040 bytes 2024-09-14T08:51:50.905 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:7100 IOs, 2770550784 bytes 2024-09-14T08:51:51.041 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:7200 IOs, 2806710272 bytes 2024-09-14T08:51:51.244 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ error_code=124 2024-09-14T08:51:51.245 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set -e 2024-09-14T08:51:51.245 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T08:51:51.245 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:51:51.245 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T08:51:51.245 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_41 2024-09-14T08:51:51.245 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image_and_enable_mirror cluster2 mirror image_41 snapshot 128M 2024-09-14T08:51:51.245 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:51:51.245 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:51:51.245 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:51:51.245 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:51:51.245 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_41 2024-09-14T08:51:51.245 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:51:51.245 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:51:51.245 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n snapshot ']' 2024-09-14T08:51:51.246 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:51:51.246 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image cluster2 mirror image_41 128M 2024-09-14T08:51:51.246 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:51:51.246 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:51:51.246 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:51:51.246 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:51:51.246 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_41 2024-09-14T08:51:51.246 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:51:51.246 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local size=128 2024-09-14T08:51:51.246 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n 128M ']' 2024-09-14T08:51:51.246 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ size=128M 2024-09-14T08:51:51.246 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:51:51.246 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_41 2024-09-14T08:51:51.298 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T08:51:51.298 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:51:51.298 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' image = image ']' 2024-09-14T08:51:51.298 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:51:51.298 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ enable_mirror cluster2 mirror image_41 snapshot 2024-09-14T08:51:51.298 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:51:51.298 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:51:51.299 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_41 2024-09-14T08:51:51.299 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:51:51.299 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_41 snapshot 2024-09-14T08:51:52.213 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Mirroring enabled 2024-09-14T08:51:52.220 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 info mirror/image_41 2024-09-14T08:51:52.262 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_41': 2024-09-14T08:51:52.262 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T08:51:52.262 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:51:52.263 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 1 2024-09-14T08:51:52.263 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 146cb4bb585e 2024-09-14T08:51:52.263 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.146cb4bb585e 2024-09-14T08:51:52.263 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:51:52.263 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering 2024-09-14T08:51:52.263 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:51:52.263 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:51:52.263 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:51:51 2024 2024-09-14T08:51:52.263 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:51:51 2024 2024-09-14T08:51:52.263 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:51:51 2024 2024-09-14T08:51:52.263 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:51:52.263 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:51:52.263 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: b734de70-db13-448a-a651-6020a282fb17 2024-09-14T08:51:52.264 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: true 2024-09-14T08:51:52.266 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n '' ']' 2024-09-14T08:51:52.266 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ stress_write_image cluster2 mirror image_41 2024-09-14T08:51:52.266 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:51:52.266 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:51:52.266 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_41 2024-09-14T08:51:52.266 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T08:51:52.268 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local duration=13 2024-09-14T08:51:52.268 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set +e 2024-09-14T08:51:52.268 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ timeout 13s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_41 --debug-rbd=20 --debug-journaler=20 2024-09-14T08:51:52.462 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:100 IOs, 35305472 bytes 2024-09-14T08:51:52.573 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:200 IOs, 42638336 bytes 2024-09-14T08:51:52.797 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:300 IOs, 115003392 bytes 2024-09-14T08:51:53.074 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:400 IOs, 180868096 bytes 2024-09-14T08:51:53.369 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:500 IOs, 259931136 bytes 2024-09-14T08:51:53.533 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:600 IOs, 298762240 bytes 2024-09-14T08:51:53.685 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:700 IOs, 323682304 bytes 2024-09-14T08:51:53.919 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:800 IOs, 385076224 bytes 2024-09-14T08:51:54.122 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:900 IOs, 434460672 bytes 2024-09-14T08:51:54.276 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1000 IOs, 464046080 bytes 2024-09-14T08:51:54.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1100 IOs, 464674816 bytes 2024-09-14T08:51:54.574 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1200 IOs, 522361856 bytes 2024-09-14T08:51:54.755 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1300 IOs, 570402816 bytes 2024-09-14T08:51:55.011 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1400 IOs, 629163008 bytes 2024-09-14T08:51:55.200 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1500 IOs, 671229952 bytes 2024-09-14T08:51:55.304 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1600 IOs, 678877184 bytes 2024-09-14T08:51:55.456 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1700 IOs, 704684032 bytes 2024-09-14T08:51:55.579 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1800 IOs, 734234624 bytes 2024-09-14T08:51:55.790 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1900 IOs, 776072192 bytes 2024-09-14T08:51:55.950 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2000 IOs, 807189504 bytes 2024-09-14T08:51:56.165 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2100 IOs, 860906496 bytes 2024-09-14T08:51:56.321 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2200 IOs, 896808960 bytes 2024-09-14T08:51:56.491 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2300 IOs, 925178880 bytes 2024-09-14T08:51:56.745 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2400 IOs, 991324160 bytes 2024-09-14T08:51:56.872 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2500 IOs, 1007947776 bytes 2024-09-14T08:51:57.230 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2600 IOs, 1119964160 bytes 2024-09-14T08:51:57.480 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2700 IOs, 1195454464 bytes 2024-09-14T08:51:57.624 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2800 IOs, 1216540672 bytes 2024-09-14T08:51:57.870 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2900 IOs, 1299203072 bytes 2024-09-14T08:51:58.182 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3000 IOs, 1375222784 bytes 2024-09-14T08:51:58.459 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3100 IOs, 1451664384 bytes 2024-09-14T08:51:58.569 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3200 IOs, 1466993664 bytes 2024-09-14T08:51:58.654 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3300 IOs, 1467601920 bytes 2024-09-14T08:51:58.763 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3400 IOs, 1480181760 bytes 2024-09-14T08:51:58.964 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3500 IOs, 1540203520 bytes 2024-09-14T08:51:59.078 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3600 IOs, 1540805632 bytes 2024-09-14T08:51:59.290 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3700 IOs, 1597276160 bytes 2024-09-14T08:51:59.433 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3800 IOs, 1631929344 bytes 2024-09-14T08:51:59.715 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3900 IOs, 1701566464 bytes 2024-09-14T08:52:00.049 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4000 IOs, 1806384128 bytes 2024-09-14T08:52:00.406 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4100 IOs, 1923909632 bytes 2024-09-14T08:52:00.656 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4200 IOs, 1986554880 bytes 2024-09-14T08:52:00.756 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4300 IOs, 1990828032 bytes 2024-09-14T08:52:00.977 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4400 IOs, 2058472448 bytes 2024-09-14T08:52:01.339 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4500 IOs, 2162027520 bytes 2024-09-14T08:52:01.646 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4600 IOs, 2245538816 bytes 2024-09-14T08:52:01.970 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4700 IOs, 2364314624 bytes 2024-09-14T08:52:02.296 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4800 IOs, 2432066560 bytes 2024-09-14T08:52:02.496 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4900 IOs, 2481213440 bytes 2024-09-14T08:52:02.617 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5000 IOs, 2497563648 bytes 2024-09-14T08:52:02.834 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5100 IOs, 2557863936 bytes 2024-09-14T08:52:03.010 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5200 IOs, 2589374464 bytes 2024-09-14T08:52:03.161 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5300 IOs, 2619551744 bytes 2024-09-14T08:52:03.263 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5400 IOs, 2623471616 bytes 2024-09-14T08:52:03.470 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5500 IOs, 2672092160 bytes 2024-09-14T08:52:03.603 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5600 IOs, 2694329344 bytes 2024-09-14T08:52:03.741 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5700 IOs, 2717164544 bytes 2024-09-14T08:52:03.901 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5800 IOs, 2749195264 bytes 2024-09-14T08:52:04.185 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5900 IOs, 2829952000 bytes 2024-09-14T08:52:04.371 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6000 IOs, 2873043968 bytes 2024-09-14T08:52:04.600 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6100 IOs, 2928365568 bytes 2024-09-14T08:52:04.799 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6200 IOs, 2972511232 bytes 2024-09-14T08:52:04.968 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6300 IOs, 3008805888 bytes 2024-09-14T08:52:05.238 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6400 IOs, 3084235776 bytes 2024-09-14T08:52:05.274 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ error_code=124 2024-09-14T08:52:05.274 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set -e 2024-09-14T08:52:05.274 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T08:52:05.274 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:52:05.274 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T08:52:05.274 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_42 2024-09-14T08:52:05.274 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image_and_enable_mirror cluster2 mirror image_42 snapshot 128M 2024-09-14T08:52:05.275 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:52:05.275 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:52:05.275 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:52:05.275 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:52:05.275 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_42 2024-09-14T08:52:05.275 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:52:05.275 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:52:05.275 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n snapshot ']' 2024-09-14T08:52:05.275 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:52:05.275 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image cluster2 mirror image_42 128M 2024-09-14T08:52:05.275 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:52:05.275 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:52:05.276 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:52:05.276 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:52:05.276 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_42 2024-09-14T08:52:05.276 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:52:05.276 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local size=128 2024-09-14T08:52:05.276 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n 128M ']' 2024-09-14T08:52:05.276 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ size=128M 2024-09-14T08:52:05.276 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:52:05.276 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_42 2024-09-14T08:52:05.323 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T08:52:05.324 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:52:05.324 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' image = image ']' 2024-09-14T08:52:05.324 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:52:05.324 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ enable_mirror cluster2 mirror image_42 snapshot 2024-09-14T08:52:05.324 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:52:05.324 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:52:05.324 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_42 2024-09-14T08:52:05.324 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:52:05.324 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_42 snapshot 2024-09-14T08:52:06.251 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Mirroring enabled 2024-09-14T08:52:06.258 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 info mirror/image_42 2024-09-14T08:52:06.302 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_42': 2024-09-14T08:52:06.302 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T08:52:06.303 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:52:06.303 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 1 2024-09-14T08:52:06.303 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 1478254fda12 2024-09-14T08:52:06.303 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.1478254fda12 2024-09-14T08:52:06.303 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:52:06.303 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering 2024-09-14T08:52:06.303 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:52:06.303 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:52:06.303 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:52:05 2024 2024-09-14T08:52:06.303 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:52:05 2024 2024-09-14T08:52:06.303 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:52:05 2024 2024-09-14T08:52:06.303 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:52:06.303 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:52:06.304 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: 071d6746-d644-418c-9496-15e5e66ef4f1 2024-09-14T08:52:06.304 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: true 2024-09-14T08:52:06.306 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n '' ']' 2024-09-14T08:52:06.306 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ stress_write_image cluster2 mirror image_42 2024-09-14T08:52:06.306 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:52:06.306 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:52:06.307 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_42 2024-09-14T08:52:06.307 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T08:52:06.308 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local duration=9 2024-09-14T08:52:06.308 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set +e 2024-09-14T08:52:06.308 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ timeout 9s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_42 --debug-rbd=20 --debug-journaler=20 2024-09-14T08:52:06.453 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:100 IOs, 33536000 bytes 2024-09-14T08:52:06.749 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:200 IOs, 98699264 bytes 2024-09-14T08:52:06.941 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:300 IOs, 148247552 bytes 2024-09-14T08:52:07.022 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:400 IOs, 153927680 bytes 2024-09-14T08:52:07.152 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:500 IOs, 170974208 bytes 2024-09-14T08:52:07.329 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:600 IOs, 214013952 bytes 2024-09-14T08:52:07.448 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:700 IOs, 225068032 bytes 2024-09-14T08:52:07.588 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:800 IOs, 249211904 bytes 2024-09-14T08:52:07.807 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:900 IOs, 313042944 bytes 2024-09-14T08:52:08.085 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1000 IOs, 394059776 bytes 2024-09-14T08:52:08.321 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1100 IOs, 453596160 bytes 2024-09-14T08:52:08.415 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1200 IOs, 456842240 bytes 2024-09-14T08:52:08.575 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1300 IOs, 491097088 bytes 2024-09-14T08:52:08.786 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1400 IOs, 546425856 bytes 2024-09-14T08:52:08.901 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1500 IOs, 558597120 bytes 2024-09-14T08:52:09.139 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1600 IOs, 623195136 bytes 2024-09-14T08:52:09.240 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1700 IOs, 631876608 bytes 2024-09-14T08:52:09.417 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1800 IOs, 672016384 bytes 2024-09-14T08:52:09.643 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1900 IOs, 733419520 bytes 2024-09-14T08:52:09.892 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2000 IOs, 817950720 bytes 2024-09-14T08:52:10.237 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2100 IOs, 906048512 bytes 2024-09-14T08:52:10.512 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2200 IOs, 990793728 bytes 2024-09-14T08:52:10.689 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2300 IOs, 1026428928 bytes 2024-09-14T08:52:11.009 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2400 IOs, 1126384640 bytes 2024-09-14T08:52:11.138 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2500 IOs, 1146131456 bytes 2024-09-14T08:52:11.376 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2600 IOs, 1212809216 bytes 2024-09-14T08:52:11.495 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2700 IOs, 1228916736 bytes 2024-09-14T08:52:11.694 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2800 IOs, 1275323392 bytes 2024-09-14T08:52:11.783 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2900 IOs, 1283958784 bytes 2024-09-14T08:52:11.924 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3000 IOs, 1318164480 bytes 2024-09-14T08:52:12.095 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3100 IOs, 1338414080 bytes 2024-09-14T08:52:12.177 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3200 IOs, 1341099008 bytes 2024-09-14T08:52:12.280 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3300 IOs, 1351988224 bytes 2024-09-14T08:52:12.414 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3400 IOs, 1373597696 bytes 2024-09-14T08:52:12.645 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3500 IOs, 1436218368 bytes 2024-09-14T08:52:12.725 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3600 IOs, 1436841984 bytes 2024-09-14T08:52:12.829 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3700 IOs, 1455003648 bytes 2024-09-14T08:52:12.991 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3800 IOs, 1497625600 bytes 2024-09-14T08:52:13.257 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3900 IOs, 1553026048 bytes 2024-09-14T08:52:13.365 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4000 IOs, 1565271040 bytes 2024-09-14T08:52:13.557 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4100 IOs, 1613302784 bytes 2024-09-14T08:52:13.808 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4200 IOs, 1681459200 bytes 2024-09-14T08:52:13.959 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4300 IOs, 1709831168 bytes 2024-09-14T08:52:14.113 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4400 IOs, 1746131968 bytes 2024-09-14T08:52:14.227 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4500 IOs, 1759277056 bytes 2024-09-14T08:52:14.419 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4600 IOs, 1800177664 bytes 2024-09-14T08:52:14.534 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4700 IOs, 1814966272 bytes 2024-09-14T08:52:14.894 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4800 IOs, 1931667456 bytes 2024-09-14T08:52:15.160 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4900 IOs, 2028862464 bytes 2024-09-14T08:52:15.314 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ error_code=124 2024-09-14T08:52:15.314 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set -e 2024-09-14T08:52:15.314 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T08:52:15.314 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:52:15.314 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T08:52:15.314 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_43 2024-09-14T08:52:15.315 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image_and_enable_mirror cluster2 mirror image_43 snapshot 128M 2024-09-14T08:52:15.315 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:52:15.315 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:52:15.315 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:52:15.315 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:52:15.315 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_43 2024-09-14T08:52:15.315 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:52:15.315 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:52:15.315 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n snapshot ']' 2024-09-14T08:52:15.315 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:52:15.315 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image cluster2 mirror image_43 128M 2024-09-14T08:52:15.315 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:52:15.315 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:52:15.316 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:52:15.316 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:52:15.316 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_43 2024-09-14T08:52:15.316 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:52:15.316 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local size=128 2024-09-14T08:52:15.316 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n 128M ']' 2024-09-14T08:52:15.316 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ size=128M 2024-09-14T08:52:15.316 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:52:15.316 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_43 2024-09-14T08:52:15.379 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T08:52:15.379 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:52:15.379 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' image = image ']' 2024-09-14T08:52:15.379 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:52:15.379 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ enable_mirror cluster2 mirror image_43 snapshot 2024-09-14T08:52:15.379 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:52:15.379 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:52:15.379 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_43 2024-09-14T08:52:15.379 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:52:15.379 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_43 snapshot 2024-09-14T08:52:16.261 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Mirroring enabled 2024-09-14T08:52:16.267 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 info mirror/image_43 2024-09-14T08:52:16.309 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_43': 2024-09-14T08:52:16.309 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T08:52:16.309 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:52:16.309 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 1 2024-09-14T08:52:16.309 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 14842eb0ff11 2024-09-14T08:52:16.309 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.14842eb0ff11 2024-09-14T08:52:16.309 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:52:16.309 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering 2024-09-14T08:52:16.310 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:52:16.310 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:52:16.310 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:52:15 2024 2024-09-14T08:52:16.310 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:52:15 2024 2024-09-14T08:52:16.310 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:52:15 2024 2024-09-14T08:52:16.310 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:52:16.310 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:52:16.310 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: 3589cded-f286-4a62-a351-1f0a15e1b107 2024-09-14T08:52:16.310 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: true 2024-09-14T08:52:16.312 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n '' ']' 2024-09-14T08:52:16.313 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ stress_write_image cluster2 mirror image_43 2024-09-14T08:52:16.313 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:52:16.313 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:52:16.313 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_43 2024-09-14T08:52:16.313 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T08:52:16.314 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local duration=12 2024-09-14T08:52:16.314 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set +e 2024-09-14T08:52:16.314 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ timeout 12s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_43 --debug-rbd=20 --debug-journaler=20 2024-09-14T08:52:16.544 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:100 IOs, 43905024 bytes 2024-09-14T08:52:16.661 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:200 IOs, 58742784 bytes 2024-09-14T08:52:16.839 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:300 IOs, 98699264 bytes 2024-09-14T08:52:16.960 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:400 IOs, 131918848 bytes 2024-09-14T08:52:17.179 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:500 IOs, 177307648 bytes 2024-09-14T08:52:17.441 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:600 IOs, 247847936 bytes 2024-09-14T08:52:17.619 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:700 IOs, 294987776 bytes 2024-09-14T08:52:17.716 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:800 IOs, 295607296 bytes 2024-09-14T08:52:17.944 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:900 IOs, 358469632 bytes 2024-09-14T08:52:18.243 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1000 IOs, 441668608 bytes 2024-09-14T08:52:18.366 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1100 IOs, 459444224 bytes 2024-09-14T08:52:18.605 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1200 IOs, 522786816 bytes 2024-09-14T08:52:18.736 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1300 IOs, 543542272 bytes 2024-09-14T08:52:18.940 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1400 IOs, 602625024 bytes 2024-09-14T08:52:19.281 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1500 IOs, 706270208 bytes 2024-09-14T08:52:19.561 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1600 IOs, 787142656 bytes 2024-09-14T08:52:19.799 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1700 IOs, 849111040 bytes 2024-09-14T08:52:20.107 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1800 IOs, 951921664 bytes 2024-09-14T08:52:20.222 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1900 IOs, 962696192 bytes 2024-09-14T08:52:20.333 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2000 IOs, 975139840 bytes 2024-09-14T08:52:20.535 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2100 IOs, 1025661952 bytes 2024-09-14T08:52:20.769 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2200 IOs, 1102078976 bytes 2024-09-14T08:52:20.956 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2300 IOs, 1142700032 bytes 2024-09-14T08:52:21.126 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2400 IOs, 1172141056 bytes 2024-09-14T08:52:21.318 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2500 IOs, 1212904448 bytes 2024-09-14T08:52:21.526 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2600 IOs, 1261745152 bytes 2024-09-14T08:52:21.747 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2700 IOs, 1322918912 bytes 2024-09-14T08:52:21.995 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2800 IOs, 1387591680 bytes 2024-09-14T08:52:22.122 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2900 IOs, 1403499520 bytes 2024-09-14T08:52:22.295 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3000 IOs, 1439787008 bytes 2024-09-14T08:52:22.476 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3100 IOs, 1483765760 bytes 2024-09-14T08:52:22.644 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3200 IOs, 1518800896 bytes 2024-09-14T08:52:22.820 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3300 IOs, 1559270400 bytes 2024-09-14T08:52:23.092 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3400 IOs, 1635643392 bytes 2024-09-14T08:52:23.264 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3500 IOs, 1673110528 bytes 2024-09-14T08:52:23.352 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3600 IOs, 1673695232 bytes 2024-09-14T08:52:23.521 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3700 IOs, 1709544448 bytes 2024-09-14T08:52:23.634 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3800 IOs, 1723386880 bytes 2024-09-14T08:52:23.736 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3900 IOs, 1733038080 bytes 2024-09-14T08:52:23.880 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4000 IOs, 1758556160 bytes 2024-09-14T08:52:24.013 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4100 IOs, 1792920576 bytes 2024-09-14T08:52:24.262 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4200 IOs, 1845637120 bytes 2024-09-14T08:52:24.456 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4300 IOs, 1889938432 bytes 2024-09-14T08:52:24.565 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4400 IOs, 1903730688 bytes 2024-09-14T08:52:24.740 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4500 IOs, 1942909952 bytes 2024-09-14T08:52:24.856 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4600 IOs, 1955779584 bytes 2024-09-14T08:52:24.997 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4700 IOs, 1981073408 bytes 2024-09-14T08:52:25.129 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4800 IOs, 2013310976 bytes 2024-09-14T08:52:25.333 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4900 IOs, 2067273728 bytes 2024-09-14T08:52:25.561 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5000 IOs, 2122152960 bytes 2024-09-14T08:52:25.701 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5100 IOs, 2132417536 bytes 2024-09-14T08:52:25.780 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5200 IOs, 2133051392 bytes 2024-09-14T08:52:25.863 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5300 IOs, 2133672960 bytes 2024-09-14T08:52:26.070 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5400 IOs, 2182088704 bytes 2024-09-14T08:52:26.212 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5500 IOs, 2207355904 bytes 2024-09-14T08:52:26.309 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5600 IOs, 2215027712 bytes 2024-09-14T08:52:26.390 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5700 IOs, 2215661568 bytes 2024-09-14T08:52:26.577 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5800 IOs, 2259644416 bytes 2024-09-14T08:52:26.959 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5900 IOs, 2383981568 bytes 2024-09-14T08:52:27.073 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6000 IOs, 2392043520 bytes 2024-09-14T08:52:27.291 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6100 IOs, 2447747072 bytes 2024-09-14T08:52:27.413 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6200 IOs, 2465759232 bytes 2024-09-14T08:52:27.557 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6300 IOs, 2490779648 bytes 2024-09-14T08:52:27.851 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6400 IOs, 2573686784 bytes 2024-09-14T08:52:27.996 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6500 IOs, 2606600192 bytes 2024-09-14T08:52:28.152 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6600 IOs, 2649465856 bytes 2024-09-14T08:52:28.322 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ error_code=124 2024-09-14T08:52:28.322 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set -e 2024-09-14T08:52:28.322 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T08:52:28.322 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:52:28.322 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T08:52:28.322 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_44 2024-09-14T08:52:28.322 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image_and_enable_mirror cluster2 mirror image_44 snapshot 128M 2024-09-14T08:52:28.322 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:52:28.322 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:52:28.323 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:52:28.323 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:52:28.323 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_44 2024-09-14T08:52:28.323 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:52:28.323 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:52:28.323 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n snapshot ']' 2024-09-14T08:52:28.323 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:52:28.323 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image cluster2 mirror image_44 128M 2024-09-14T08:52:28.323 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:52:28.323 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:52:28.323 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:52:28.323 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:52:28.323 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_44 2024-09-14T08:52:28.324 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:52:28.324 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local size=128 2024-09-14T08:52:28.324 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n 128M ']' 2024-09-14T08:52:28.324 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ size=128M 2024-09-14T08:52:28.324 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:52:28.324 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_44 2024-09-14T08:52:28.371 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T08:52:28.371 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:52:28.372 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' image = image ']' 2024-09-14T08:52:28.372 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:52:28.372 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ enable_mirror cluster2 mirror image_44 snapshot 2024-09-14T08:52:28.372 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:52:28.372 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:52:28.372 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_44 2024-09-14T08:52:28.372 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:52:28.372 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_44 snapshot 2024-09-14T08:52:29.279 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Mirroring enabled 2024-09-14T08:52:29.284 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 info mirror/image_44 2024-09-14T08:52:29.325 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_44': 2024-09-14T08:52:29.325 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T08:52:29.325 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:52:29.325 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 1 2024-09-14T08:52:29.325 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 14904c52f9c3 2024-09-14T08:52:29.325 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.14904c52f9c3 2024-09-14T08:52:29.325 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:52:29.325 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering 2024-09-14T08:52:29.325 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:52:29.325 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:52:29.325 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:52:28 2024 2024-09-14T08:52:29.326 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:52:28 2024 2024-09-14T08:52:29.326 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:52:28 2024 2024-09-14T08:52:29.326 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:52:29.326 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:52:29.326 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: 484caad3-31ac-439e-a6d6-4a11118a3aed 2024-09-14T08:52:29.326 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: true 2024-09-14T08:52:29.328 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n '' ']' 2024-09-14T08:52:29.329 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ stress_write_image cluster2 mirror image_44 2024-09-14T08:52:29.329 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:52:29.329 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:52:29.329 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_44 2024-09-14T08:52:29.329 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T08:52:29.330 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local duration=7 2024-09-14T08:52:29.330 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set +e 2024-09-14T08:52:29.330 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ timeout 7s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_44 --debug-rbd=20 --debug-journaler=20 2024-09-14T08:52:29.487 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:100 IOs, 20047872 bytes 2024-09-14T08:52:29.628 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:200 IOs, 40056832 bytes 2024-09-14T08:52:29.748 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:300 IOs, 57814016 bytes 2024-09-14T08:52:30.105 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:400 IOs, 190029824 bytes 2024-09-14T08:52:30.310 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:500 IOs, 220450816 bytes 2024-09-14T08:52:30.408 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:600 IOs, 228652032 bytes 2024-09-14T08:52:30.632 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:700 IOs, 293230592 bytes 2024-09-14T08:52:30.847 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:800 IOs, 342637568 bytes 2024-09-14T08:52:30.949 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:900 IOs, 346836992 bytes 2024-09-14T08:52:31.177 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1000 IOs, 408386560 bytes 2024-09-14T08:52:31.329 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1100 IOs, 432998400 bytes 2024-09-14T08:52:31.470 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1200 IOs, 454594560 bytes 2024-09-14T08:52:31.612 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1300 IOs, 483991552 bytes 2024-09-14T08:52:31.758 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1400 IOs, 509198336 bytes 2024-09-14T08:52:32.065 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1500 IOs, 604804096 bytes 2024-09-14T08:52:32.270 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1600 IOs, 641624064 bytes 2024-09-14T08:52:32.401 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1700 IOs, 660206592 bytes 2024-09-14T08:52:32.557 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1800 IOs, 704045056 bytes 2024-09-14T08:52:32.774 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1900 IOs, 743583744 bytes 2024-09-14T08:52:32.940 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2000 IOs, 778904576 bytes 2024-09-14T08:52:33.232 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2100 IOs, 860146688 bytes 2024-09-14T08:52:33.382 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2200 IOs, 883089408 bytes 2024-09-14T08:52:33.531 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2300 IOs, 919804928 bytes 2024-09-14T08:52:33.725 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2400 IOs, 955757568 bytes 2024-09-14T08:52:33.825 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2500 IOs, 991937536 bytes 2024-09-14T08:52:34.065 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2600 IOs, 1032782848 bytes 2024-09-14T08:52:34.261 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2700 IOs, 1075580928 bytes 2024-09-14T08:52:34.405 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2800 IOs, 1098633216 bytes 2024-09-14T08:52:34.666 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2900 IOs, 1165274112 bytes 2024-09-14T08:52:34.861 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3000 IOs, 1199477760 bytes 2024-09-14T08:52:35.014 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3100 IOs, 1229825024 bytes 2024-09-14T08:52:35.119 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3200 IOs, 1237431296 bytes 2024-09-14T08:52:35.253 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3300 IOs, 1260158976 bytes 2024-09-14T08:52:35.378 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3400 IOs, 1294329856 bytes 2024-09-14T08:52:35.533 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3500 IOs, 1311987712 bytes 2024-09-14T08:52:35.860 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3600 IOs, 1412460544 bytes 2024-09-14T08:52:35.986 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3700 IOs, 1424576512 bytes 2024-09-14T08:52:36.211 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3800 IOs, 1482639360 bytes 2024-09-14T08:52:36.336 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ error_code=124 2024-09-14T08:52:36.336 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set -e 2024-09-14T08:52:36.336 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T08:52:36.336 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:52:36.336 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T08:52:36.336 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_45 2024-09-14T08:52:36.336 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image_and_enable_mirror cluster2 mirror image_45 snapshot 128M 2024-09-14T08:52:36.336 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:52:36.337 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:52:36.337 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:52:36.337 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:52:36.337 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_45 2024-09-14T08:52:36.337 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:52:36.337 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:52:36.337 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n snapshot ']' 2024-09-14T08:52:36.337 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:52:36.337 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image cluster2 mirror image_45 128M 2024-09-14T08:52:36.337 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:52:36.337 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:52:36.337 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:52:36.337 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:52:36.338 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_45 2024-09-14T08:52:36.338 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:52:36.338 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local size=128 2024-09-14T08:52:36.338 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n 128M ']' 2024-09-14T08:52:36.338 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ size=128M 2024-09-14T08:52:36.338 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:52:36.338 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_45 2024-09-14T08:52:36.404 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T08:52:36.404 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:52:36.404 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' image = image ']' 2024-09-14T08:52:36.404 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:52:36.404 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ enable_mirror cluster2 mirror image_45 snapshot 2024-09-14T08:52:36.404 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:52:36.404 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:52:36.404 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_45 2024-09-14T08:52:36.405 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:52:36.405 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_45 snapshot 2024-09-14T08:52:37.292 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Mirroring enabled 2024-09-14T08:52:37.299 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 info mirror/image_45 2024-09-14T08:52:37.339 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_45': 2024-09-14T08:52:37.339 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T08:52:37.339 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:52:37.339 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 1 2024-09-14T08:52:37.339 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 149cf4df82a 2024-09-14T08:52:37.339 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.149cf4df82a 2024-09-14T08:52:37.339 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:52:37.339 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering 2024-09-14T08:52:37.339 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:52:37.339 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:52:37.339 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:52:36 2024 2024-09-14T08:52:37.340 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:52:36 2024 2024-09-14T08:52:37.340 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:52:36 2024 2024-09-14T08:52:37.340 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:52:37.340 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:52:37.340 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: c2a67c35-feb8-43a1-95f7-a8a88b0481e9 2024-09-14T08:52:37.340 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: true 2024-09-14T08:52:37.342 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n '' ']' 2024-09-14T08:52:37.342 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ stress_write_image cluster2 mirror image_45 2024-09-14T08:52:37.342 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:52:37.343 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:52:37.343 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_45 2024-09-14T08:52:37.343 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T08:52:37.344 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local duration=7 2024-09-14T08:52:37.345 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set +e 2024-09-14T08:52:37.345 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ timeout 7s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_45 --debug-rbd=20 --debug-journaler=20 2024-09-14T08:52:37.627 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:100 IOs, 69983232 bytes 2024-09-14T08:52:37.798 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:200 IOs, 104772608 bytes 2024-09-14T08:52:37.881 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:300 IOs, 105375744 bytes 2024-09-14T08:52:38.042 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:400 IOs, 147162112 bytes 2024-09-14T08:52:38.354 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:500 IOs, 235479040 bytes 2024-09-14T08:52:38.511 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:600 IOs, 258225152 bytes 2024-09-14T08:52:38.620 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:700 IOs, 270513152 bytes 2024-09-14T08:52:38.760 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:800 IOs, 301564928 bytes 2024-09-14T08:52:38.903 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:900 IOs, 338479104 bytes 2024-09-14T08:52:39.135 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1000 IOs, 380636160 bytes 2024-09-14T08:52:39.283 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1100 IOs, 407825408 bytes 2024-09-14T08:52:39.473 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1200 IOs, 447337472 bytes 2024-09-14T08:52:39.610 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1300 IOs, 472960000 bytes 2024-09-14T08:52:39.925 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1400 IOs, 573432832 bytes 2024-09-14T08:52:40.118 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1500 IOs, 609351680 bytes 2024-09-14T08:52:40.308 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1600 IOs, 657415168 bytes 2024-09-14T08:52:40.440 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1700 IOs, 671686656 bytes 2024-09-14T08:52:40.650 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1800 IOs, 727556096 bytes 2024-09-14T08:52:40.827 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1900 IOs, 763223040 bytes 2024-09-14T08:52:40.999 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2000 IOs, 812171264 bytes 2024-09-14T08:52:41.189 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2100 IOs, 842896384 bytes 2024-09-14T08:52:41.391 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2200 IOs, 889286656 bytes 2024-09-14T08:52:41.516 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2300 IOs, 928599040 bytes 2024-09-14T08:52:41.727 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2400 IOs, 970268672 bytes 2024-09-14T08:52:42.115 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2500 IOs, 1073667072 bytes 2024-09-14T08:52:42.289 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2600 IOs, 1113915392 bytes 2024-09-14T08:52:42.455 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2700 IOs, 1152358400 bytes 2024-09-14T08:52:42.783 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2800 IOs, 1244272640 bytes 2024-09-14T08:52:42.869 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2900 IOs, 1244868608 bytes 2024-09-14T08:52:43.122 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3000 IOs, 1309935616 bytes 2024-09-14T08:52:43.304 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3100 IOs, 1349491712 bytes 2024-09-14T08:52:43.526 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3200 IOs, 1406545920 bytes 2024-09-14T08:52:43.732 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3300 IOs, 1448736768 bytes 2024-09-14T08:52:43.809 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3400 IOs, 1449360384 bytes 2024-09-14T08:52:44.044 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3500 IOs, 1509564416 bytes 2024-09-14T08:52:44.292 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3600 IOs, 1577015296 bytes 2024-09-14T08:52:44.352 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ error_code=124 2024-09-14T08:52:44.352 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set -e 2024-09-14T08:52:44.353 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T08:52:44.353 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:52:44.353 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T08:52:44.353 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_46 2024-09-14T08:52:44.353 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image_and_enable_mirror cluster2 mirror image_46 snapshot 128M 2024-09-14T08:52:44.353 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:52:44.353 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:52:44.353 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:52:44.353 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:52:44.353 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_46 2024-09-14T08:52:44.353 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:52:44.353 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:52:44.354 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n snapshot ']' 2024-09-14T08:52:44.354 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:52:44.354 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image cluster2 mirror image_46 128M 2024-09-14T08:52:44.354 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:52:44.354 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:52:44.354 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:52:44.354 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:52:44.354 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_46 2024-09-14T08:52:44.354 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:52:44.354 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local size=128 2024-09-14T08:52:44.354 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n 128M ']' 2024-09-14T08:52:44.354 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ size=128M 2024-09-14T08:52:44.354 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:52:44.355 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_46 2024-09-14T08:52:44.409 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T08:52:44.410 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:52:44.410 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' image = image ']' 2024-09-14T08:52:44.410 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:52:44.410 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ enable_mirror cluster2 mirror image_46 snapshot 2024-09-14T08:52:44.410 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:52:44.410 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:52:44.410 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_46 2024-09-14T08:52:44.410 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:52:44.410 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_46 snapshot 2024-09-14T08:52:45.303 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Mirroring enabled 2024-09-14T08:52:45.309 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 info mirror/image_46 2024-09-14T08:52:45.349 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_46': 2024-09-14T08:52:45.349 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T08:52:45.349 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:52:45.349 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 1 2024-09-14T08:52:45.349 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 14a8bfe4fab2 2024-09-14T08:52:45.349 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.14a8bfe4fab2 2024-09-14T08:52:45.349 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:52:45.349 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering 2024-09-14T08:52:45.350 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:52:45.350 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:52:45.350 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:52:44 2024 2024-09-14T08:52:45.350 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:52:44 2024 2024-09-14T08:52:45.350 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:52:44 2024 2024-09-14T08:52:45.350 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:52:45.350 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:52:45.350 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: 0f89d25d-4adb-4110-bda2-74b7704ad44b 2024-09-14T08:52:45.350 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: true 2024-09-14T08:52:45.352 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n '' ']' 2024-09-14T08:52:45.353 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ stress_write_image cluster2 mirror image_46 2024-09-14T08:52:45.353 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:52:45.353 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:52:45.353 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_46 2024-09-14T08:52:45.353 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T08:52:45.354 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local duration=7 2024-09-14T08:52:45.354 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set +e 2024-09-14T08:52:45.354 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ timeout 7s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_46 --debug-rbd=20 --debug-journaler=20 2024-09-14T08:52:45.686 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:100 IOs, 92647424 bytes 2024-09-14T08:52:45.789 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:200 IOs, 96807936 bytes 2024-09-14T08:52:45.945 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:300 IOs, 129202176 bytes 2024-09-14T08:52:46.027 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:400 IOs, 129817600 bytes 2024-09-14T08:52:46.242 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:500 IOs, 187559936 bytes 2024-09-14T08:52:46.334 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:600 IOs, 191330304 bytes 2024-09-14T08:52:46.628 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:700 IOs, 281218048 bytes 2024-09-14T08:52:46.882 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:800 IOs, 358936576 bytes 2024-09-14T08:52:47.082 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:900 IOs, 401121280 bytes 2024-09-14T08:52:47.148 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.7437640021598984) > weight (0.25) 2024-09-14T08:52:47.148 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.6391316366771124) > weight (0.25) 2024-09-14T08:52:47.148 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.3386010650257242) > weight (0.25) 2024-09-14T08:52:47.148 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.3 with skip (0.7385919866293303) > weight (0.25) 2024-09-14T08:52:47.148 INFO:tasks.rbd_mirror_thrash:waiting for 119 secs before thrashing 2024-09-14T08:52:47.486 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1000 IOs, 530128896 bytes 2024-09-14T08:52:47.627 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1100 IOs, 552072192 bytes 2024-09-14T08:52:47.711 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1200 IOs, 554784768 bytes 2024-09-14T08:52:47.846 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1300 IOs, 578568192 bytes 2024-09-14T08:52:48.073 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1400 IOs, 642086912 bytes 2024-09-14T08:52:48.186 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1500 IOs, 653514752 bytes 2024-09-14T08:52:48.374 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1600 IOs, 699759616 bytes 2024-09-14T08:52:48.542 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1700 IOs, 734539776 bytes 2024-09-14T08:52:48.723 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1800 IOs, 768627712 bytes 2024-09-14T08:52:48.897 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1900 IOs, 810097664 bytes 2024-09-14T08:52:48.930 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.0 2024-09-14T08:52:48.931 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Sent signal 15 2024-09-14T08:52:48.931 INFO:tasks.rbd_mirror_thrash:waiting for 7 secs before reviving daemons 2024-09-14T08:52:48.932 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi164.stderr:2024-09-14T08:52:48.931+0000 7f5d7084c640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 (PID: 46583) UID: 1000 2024-09-14T08:52:49.022 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2000 IOs, 827420672 bytes 2024-09-14T08:52:49.144 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2100 IOs, 838697984 bytes 2024-09-14T08:52:49.368 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2200 IOs, 890249216 bytes 2024-09-14T08:52:49.581 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2300 IOs, 945453056 bytes 2024-09-14T08:52:49.700 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2400 IOs, 962260992 bytes 2024-09-14T08:52:49.865 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2500 IOs, 1009245184 bytes 2024-09-14T08:52:49.988 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2600 IOs, 1015436288 bytes 2024-09-14T08:52:50.165 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2700 IOs, 1055376384 bytes 2024-09-14T08:52:50.398 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2800 IOs, 1114054656 bytes 2024-09-14T08:52:50.483 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2900 IOs, 1116505088 bytes 2024-09-14T08:52:50.698 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3000 IOs, 1173478400 bytes 2024-09-14T08:52:50.830 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3100 IOs, 1194097664 bytes 2024-09-14T08:52:50.994 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3200 IOs, 1228542976 bytes 2024-09-14T08:52:51.117 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3300 IOs, 1245604864 bytes 2024-09-14T08:52:51.356 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3400 IOs, 1310120960 bytes 2024-09-14T08:52:51.564 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3500 IOs, 1354718208 bytes 2024-09-14T08:52:51.711 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3600 IOs, 1381740544 bytes 2024-09-14T08:52:51.876 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3700 IOs, 1417127936 bytes 2024-09-14T08:52:52.122 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3800 IOs, 1486984192 bytes 2024-09-14T08:52:52.349 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3900 IOs, 1546529792 bytes 2024-09-14T08:52:52.361 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ error_code=124 2024-09-14T08:52:52.361 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set -e 2024-09-14T08:52:52.361 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T08:52:52.361 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:52:52.361 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T08:52:52.361 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_47 2024-09-14T08:52:52.361 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image_and_enable_mirror cluster2 mirror image_47 snapshot 128M 2024-09-14T08:52:52.361 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:52:52.361 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:52:52.361 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:52:52.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:52:52.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_47 2024-09-14T08:52:52.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:52:52.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:52:52.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n snapshot ']' 2024-09-14T08:52:52.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:52:52.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image cluster2 mirror image_47 128M 2024-09-14T08:52:52.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:52:52.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:52:52.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:52:52.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:52:52.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_47 2024-09-14T08:52:52.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:52:52.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local size=128 2024-09-14T08:52:52.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n 128M ']' 2024-09-14T08:52:52.363 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ size=128M 2024-09-14T08:52:52.363 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:52:52.363 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_47 2024-09-14T08:52:52.413 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T08:52:52.413 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:52:52.413 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' image = image ']' 2024-09-14T08:52:52.413 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:52:52.413 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ enable_mirror cluster2 mirror image_47 snapshot 2024-09-14T08:52:52.413 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:52:52.413 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:52:52.413 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_47 2024-09-14T08:52:52.413 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:52:52.413 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_47 snapshot 2024-09-14T08:52:53.317 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Mirroring enabled 2024-09-14T08:52:53.324 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 info mirror/image_47 2024-09-14T08:52:53.366 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_47': 2024-09-14T08:52:53.366 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T08:52:53.366 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:52:53.366 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 1 2024-09-14T08:52:53.366 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 14b421aa66f9 2024-09-14T08:52:53.366 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.14b421aa66f9 2024-09-14T08:52:53.366 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:52:53.366 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering 2024-09-14T08:52:53.367 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:52:53.367 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:52:53.367 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:52:52 2024 2024-09-14T08:52:53.367 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:52:52 2024 2024-09-14T08:52:53.367 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:52:52 2024 2024-09-14T08:52:53.367 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:52:53.367 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:52:53.367 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: 0654f4ee-c91c-4318-964b-9a7c32deb872 2024-09-14T08:52:53.367 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: true 2024-09-14T08:52:53.372 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n '' ']' 2024-09-14T08:52:53.372 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ stress_write_image cluster2 mirror image_47 2024-09-14T08:52:53.372 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:52:53.373 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:52:53.373 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_47 2024-09-14T08:52:53.373 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T08:52:53.373 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local duration=7 2024-09-14T08:52:53.373 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set +e 2024-09-14T08:52:53.373 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ timeout 7s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_47 --debug-rbd=20 --debug-journaler=20 2024-09-14T08:52:53.509 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:100 IOs, 4099072 bytes 2024-09-14T08:52:53.737 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:200 IOs, 64354304 bytes 2024-09-14T08:52:53.814 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:300 IOs, 64973824 bytes 2024-09-14T08:52:53.946 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:400 IOs, 88801280 bytes 2024-09-14T08:52:54.104 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:500 IOs, 123660288 bytes 2024-09-14T08:52:54.495 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:600 IOs, 244674560 bytes 2024-09-14T08:52:54.581 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:700 IOs, 248522752 bytes 2024-09-14T08:52:54.785 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:800 IOs, 300161024 bytes 2024-09-14T08:52:54.969 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:900 IOs, 350041088 bytes 2024-09-14T08:52:55.222 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1000 IOs, 412641280 bytes 2024-09-14T08:52:55.402 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1100 IOs, 467349504 bytes 2024-09-14T08:52:55.536 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1200 IOs, 469331968 bytes 2024-09-14T08:52:55.659 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1300 IOs, 499791872 bytes 2024-09-14T08:52:55.842 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1400 IOs, 528572416 bytes 2024-09-14T08:52:55.932 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.0 2024-09-14T08:52:55.932 INFO:teuthology.orchestra.run:waiting for 600 2024-09-14T08:52:55.933 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.0 2024-09-14T08:52:55.933 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Restarting daemon 2024-09-14T08:52:55.934 DEBUG:teuthology.orchestra.run.smithi164:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 2024-09-14T08:52:55.937 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Started 2024-09-14T08:52:55.937 INFO:tasks.rbd_mirror_thrash:waiting for 117 secs before thrashing 2024-09-14T08:52:55.985 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1500 IOs, 553262080 bytes 2024-09-14T08:52:56.132 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1600 IOs, 573725696 bytes 2024-09-14T08:52:56.458 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1700 IOs, 698737664 bytes 2024-09-14T08:52:56.873 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1800 IOs, 817226752 bytes 2024-09-14T08:52:57.015 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1900 IOs, 831188992 bytes 2024-09-14T08:52:57.183 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2000 IOs, 863183872 bytes 2024-09-14T08:52:57.397 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2100 IOs, 913532928 bytes 2024-09-14T08:52:57.577 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2200 IOs, 955228160 bytes 2024-09-14T08:52:57.748 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2300 IOs, 989457408 bytes 2024-09-14T08:52:57.920 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2400 IOs, 1022022656 bytes 2024-09-14T08:52:58.070 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2500 IOs, 1056606208 bytes 2024-09-14T08:52:58.327 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2600 IOs, 1122552832 bytes 2024-09-14T08:52:58.488 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2700 IOs, 1155686400 bytes 2024-09-14T08:52:58.632 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2800 IOs, 1185069056 bytes 2024-09-14T08:52:58.919 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2900 IOs, 1252952064 bytes 2024-09-14T08:52:59.132 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3000 IOs, 1307535360 bytes 2024-09-14T08:52:59.378 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3100 IOs, 1376302080 bytes 2024-09-14T08:52:59.560 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3200 IOs, 1429889024 bytes 2024-09-14T08:52:59.830 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3300 IOs, 1492711424 bytes 2024-09-14T08:52:59.985 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3400 IOs, 1518708736 bytes 2024-09-14T08:53:00.247 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3500 IOs, 1616177152 bytes 2024-09-14T08:53:00.378 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ error_code=124 2024-09-14T08:53:00.379 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set -e 2024-09-14T08:53:00.379 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T08:53:00.379 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:53:00.379 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T08:53:00.379 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_48 2024-09-14T08:53:00.379 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image_and_enable_mirror cluster2 mirror image_48 snapshot 128M 2024-09-14T08:53:00.379 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:53:00.379 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:53:00.379 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:53:00.380 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:53:00.380 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_48 2024-09-14T08:53:00.380 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:53:00.380 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:53:00.380 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n snapshot ']' 2024-09-14T08:53:00.380 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:53:00.380 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image cluster2 mirror image_48 128M 2024-09-14T08:53:00.380 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:53:00.380 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:53:00.380 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:53:00.380 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:53:00.380 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_48 2024-09-14T08:53:00.380 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:53:00.381 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local size=128 2024-09-14T08:53:00.381 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n 128M ']' 2024-09-14T08:53:00.381 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ size=128M 2024-09-14T08:53:00.381 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:53:00.381 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_48 2024-09-14T08:53:00.429 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T08:53:00.429 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:53:00.429 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' image = image ']' 2024-09-14T08:53:00.429 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:53:00.429 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ enable_mirror cluster2 mirror image_48 snapshot 2024-09-14T08:53:00.429 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:53:00.429 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:53:00.429 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_48 2024-09-14T08:53:00.429 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:53:00.429 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_48 snapshot 2024-09-14T08:53:01.331 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Mirroring enabled 2024-09-14T08:53:01.338 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 info mirror/image_48 2024-09-14T08:53:01.382 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_48': 2024-09-14T08:53:01.437 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T08:53:01.437 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:53:01.437 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 1 2024-09-14T08:53:01.437 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 14c93b03b2c6 2024-09-14T08:53:01.437 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.14c93b03b2c6 2024-09-14T08:53:01.437 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:53:01.437 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering 2024-09-14T08:53:01.437 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:53:01.438 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:53:01.438 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:53:00 2024 2024-09-14T08:53:01.438 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:53:00 2024 2024-09-14T08:53:01.438 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:53:00 2024 2024-09-14T08:53:01.438 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:53:01.438 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:53:01.438 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: d56603da-84e8-4df7-8c5e-01193a0d3227 2024-09-14T08:53:01.438 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: true 2024-09-14T08:53:01.439 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n '' ']' 2024-09-14T08:53:01.440 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ stress_write_image cluster2 mirror image_48 2024-09-14T08:53:01.440 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:53:01.440 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:53:01.440 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_48 2024-09-14T08:53:01.440 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T08:53:01.440 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local duration=8 2024-09-14T08:53:01.440 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set +e 2024-09-14T08:53:01.440 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ timeout 8s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_48 --debug-rbd=20 --debug-journaler=20 2024-09-14T08:53:01.609 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:100 IOs, 37336064 bytes 2024-09-14T08:53:01.797 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:200 IOs, 82027520 bytes 2024-09-14T08:53:01.971 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:300 IOs, 118383616 bytes 2024-09-14T08:53:02.234 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:400 IOs, 193219584 bytes 2024-09-14T08:53:02.366 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:500 IOs, 211592192 bytes 2024-09-14T08:53:02.506 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:600 IOs, 236244992 bytes 2024-09-14T08:53:02.654 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:700 IOs, 262945792 bytes 2024-09-14T08:53:02.764 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:800 IOs, 276530176 bytes 2024-09-14T08:53:02.944 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:900 IOs, 316334080 bytes 2024-09-14T08:53:03.117 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1000 IOs, 353315840 bytes 2024-09-14T08:53:03.230 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1100 IOs, 372795392 bytes 2024-09-14T08:53:03.413 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1200 IOs, 420119552 bytes 2024-09-14T08:53:03.530 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1300 IOs, 420723712 bytes 2024-09-14T08:53:03.721 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1400 IOs, 472384512 bytes 2024-09-14T08:53:03.964 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1500 IOs, 530488320 bytes 2024-09-14T08:53:04.133 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1600 IOs, 564900864 bytes 2024-09-14T08:53:04.311 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1700 IOs, 607202304 bytes 2024-09-14T08:53:04.514 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1800 IOs, 657961984 bytes 2024-09-14T08:53:04.660 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1900 IOs, 691181568 bytes 2024-09-14T08:53:04.830 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2000 IOs, 718173184 bytes 2024-09-14T08:53:04.972 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2100 IOs, 744290304 bytes 2024-09-14T08:53:05.238 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2200 IOs, 816408576 bytes 2024-09-14T08:53:05.363 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2300 IOs, 834742272 bytes 2024-09-14T08:53:05.443 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2400 IOs, 835358720 bytes 2024-09-14T08:53:05.594 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2500 IOs, 877090816 bytes 2024-09-14T08:53:05.838 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2600 IOs, 934819840 bytes 2024-09-14T08:53:06.034 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2700 IOs, 978708480 bytes 2024-09-14T08:53:06.284 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2800 IOs, 1049411584 bytes 2024-09-14T08:53:06.596 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2900 IOs, 1134987264 bytes 2024-09-14T08:53:06.796 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3000 IOs, 1184239616 bytes 2024-09-14T08:53:07.091 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3100 IOs, 1269381120 bytes 2024-09-14T08:53:07.197 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3200 IOs, 1272892416 bytes 2024-09-14T08:53:07.304 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3300 IOs, 1285308416 bytes 2024-09-14T08:53:07.444 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3400 IOs, 1309580288 bytes 2024-09-14T08:53:07.577 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3500 IOs, 1331532800 bytes 2024-09-14T08:53:07.734 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3600 IOs, 1354035200 bytes 2024-09-14T08:53:07.925 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3700 IOs, 1399446528 bytes 2024-09-14T08:53:08.077 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3800 IOs, 1432539136 bytes 2024-09-14T08:53:08.331 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3900 IOs, 1500131328 bytes 2024-09-14T08:53:08.565 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4000 IOs, 1543056384 bytes 2024-09-14T08:53:08.707 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4100 IOs, 1567484928 bytes 2024-09-14T08:53:08.841 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4200 IOs, 1589145600 bytes 2024-09-14T08:53:08.973 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4300 IOs, 1618561024 bytes 2024-09-14T08:53:09.160 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4400 IOs, 1665496064 bytes 2024-09-14T08:53:09.352 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4500 IOs, 1695680512 bytes 2024-09-14T08:53:09.394 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ error_code=124 2024-09-14T08:53:09.394 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set -e 2024-09-14T08:53:09.394 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T08:53:09.394 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:53:09.394 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T08:53:09.394 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_49 2024-09-14T08:53:09.395 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image_and_enable_mirror cluster2 mirror image_49 snapshot 128M 2024-09-14T08:53:09.395 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:53:09.395 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:53:09.395 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:53:09.395 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:53:09.395 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_49 2024-09-14T08:53:09.395 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:53:09.395 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:53:09.395 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n snapshot ']' 2024-09-14T08:53:09.395 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:53:09.395 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image cluster2 mirror image_49 128M 2024-09-14T08:53:09.395 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:53:09.395 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:53:09.395 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:53:09.396 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:53:09.396 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_49 2024-09-14T08:53:09.396 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:53:09.396 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local size=128 2024-09-14T08:53:09.396 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n 128M ']' 2024-09-14T08:53:09.396 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ size=128M 2024-09-14T08:53:09.396 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:53:09.396 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_49 2024-09-14T08:53:09.445 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T08:53:09.445 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:53:09.445 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' image = image ']' 2024-09-14T08:53:09.445 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:53:09.445 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ enable_mirror cluster2 mirror image_49 snapshot 2024-09-14T08:53:09.445 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:53:09.446 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:53:09.446 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_49 2024-09-14T08:53:09.446 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:53:09.446 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_49 snapshot 2024-09-14T08:53:10.340 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Mirroring enabled 2024-09-14T08:53:10.346 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 info mirror/image_49 2024-09-14T08:53:10.389 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_49': 2024-09-14T08:53:10.389 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T08:53:10.389 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:53:10.390 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 1 2024-09-14T08:53:10.390 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 14d5f9129e58 2024-09-14T08:53:10.390 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.14d5f9129e58 2024-09-14T08:53:10.390 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:53:10.390 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering 2024-09-14T08:53:10.390 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:53:10.390 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:53:10.390 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:53:09 2024 2024-09-14T08:53:10.390 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:53:09 2024 2024-09-14T08:53:10.390 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:53:09 2024 2024-09-14T08:53:10.390 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:53:10.390 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:53:10.390 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: e852057c-0945-4c55-b535-c147c24f9585 2024-09-14T08:53:10.391 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: true 2024-09-14T08:53:10.393 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n '' ']' 2024-09-14T08:53:10.393 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ stress_write_image cluster2 mirror image_49 2024-09-14T08:53:10.393 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:53:10.393 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:53:10.393 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_49 2024-09-14T08:53:10.393 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T08:53:10.395 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local duration=6 2024-09-14T08:53:10.395 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set +e 2024-09-14T08:53:10.395 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ timeout 6s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_49 --debug-rbd=20 --debug-journaler=20 2024-09-14T08:53:10.684 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:100 IOs, 70770688 bytes 2024-09-14T08:53:10.816 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:200 IOs, 89565184 bytes 2024-09-14T08:53:10.961 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:300 IOs, 115733504 bytes 2024-09-14T08:53:11.109 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:400 IOs, 140535808 bytes 2024-09-14T08:53:11.464 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:500 IOs, 251680768 bytes 2024-09-14T08:53:11.600 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:600 IOs, 276105216 bytes 2024-09-14T08:53:11.934 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:700 IOs, 377900032 bytes 2024-09-14T08:53:12.078 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:800 IOs, 418978816 bytes 2024-09-14T08:53:12.235 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:900 IOs, 431271936 bytes 2024-09-14T08:53:12.374 INFO:tasks.ceph.cluster2.osd.2.smithi164.stderr:2024-09-14T08:53:12.372+0000 7fe32301f640 -1 reset not still connected to 0x55e755eb7790 2024-09-14T08:53:12.376 INFO:tasks.ceph.cluster2.osd.2.smithi164.stderr:2024-09-14T08:53:12.374+0000 7fe32301f640 -1 reset not still connected to 0x55e7573ad860 2024-09-14T08:53:12.397 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1000 IOs, 472712192 bytes 2024-09-14T08:53:12.644 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1100 IOs, 527002624 bytes 2024-09-14T08:53:12.890 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1200 IOs, 588644352 bytes 2024-09-14T08:53:13.083 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1300 IOs, 635449344 bytes 2024-09-14T08:53:13.244 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1400 IOs, 666994688 bytes 2024-09-14T08:53:13.423 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1500 IOs, 706717696 bytes 2024-09-14T08:53:13.727 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1600 IOs, 810316800 bytes 2024-09-14T08:53:13.953 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1700 IOs, 849611776 bytes 2024-09-14T08:53:14.116 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1800 IOs, 883274752 bytes 2024-09-14T08:53:14.247 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1900 IOs, 902763520 bytes 2024-09-14T08:53:14.577 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2000 IOs, 1004362752 bytes 2024-09-14T08:53:14.839 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2100 IOs, 1075805184 bytes 2024-09-14T08:53:15.029 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2200 IOs, 1118179328 bytes 2024-09-14T08:53:15.117 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2300 IOs, 1143983104 bytes 2024-09-14T08:53:15.292 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2400 IOs, 1158942720 bytes 2024-09-14T08:53:15.437 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2500 IOs, 1183577088 bytes 2024-09-14T08:53:15.731 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2600 IOs, 1267527680 bytes 2024-09-14T08:53:15.919 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2700 IOs, 1311444992 bytes 2024-09-14T08:53:16.057 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2800 IOs, 1333720064 bytes 2024-09-14T08:53:16.266 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2900 IOs, 1400292352 bytes 2024-09-14T08:53:16.400 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ error_code=124 2024-09-14T08:53:16.401 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set -e 2024-09-14T08:53:16.401 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T08:53:16.401 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:53:16.401 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T08:53:16.401 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_50 2024-09-14T08:53:16.401 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image_and_enable_mirror cluster2 mirror image_50 snapshot 128M 2024-09-14T08:53:16.401 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:53:16.401 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:53:16.401 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:53:16.401 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:53:16.401 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_50 2024-09-14T08:53:16.402 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:53:16.402 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:53:16.402 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n snapshot ']' 2024-09-14T08:53:16.402 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:53:16.402 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image cluster2 mirror image_50 128M 2024-09-14T08:53:16.402 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:53:16.402 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:53:16.402 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:53:16.402 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:53:16.402 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_50 2024-09-14T08:53:16.402 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:53:16.402 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local size=128 2024-09-14T08:53:16.402 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n 128M ']' 2024-09-14T08:53:16.402 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ size=128M 2024-09-14T08:53:16.402 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T08:53:16.403 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_50 2024-09-14T08:53:16.456 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T08:53:16.457 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:53:16.457 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' image = image ']' 2024-09-14T08:53:16.457 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:53:16.457 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ enable_mirror cluster2 mirror image_50 snapshot 2024-09-14T08:53:16.457 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:53:16.457 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:53:16.457 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_50 2024-09-14T08:53:16.457 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T08:53:16.457 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_50 snapshot 2024-09-14T08:53:16.541 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Mirroring enabled 2024-09-14T08:53:16.547 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 info mirror/image_50 2024-09-14T08:53:16.585 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_50': 2024-09-14T08:53:16.585 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T08:53:16.586 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:53:16.586 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 1 2024-09-14T08:53:16.586 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 14e116cae513 2024-09-14T08:53:16.586 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.14e116cae513 2024-09-14T08:53:16.586 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:53:16.586 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering 2024-09-14T08:53:16.586 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:53:16.586 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:53:16.586 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:53:16 2024 2024-09-14T08:53:16.586 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:53:16 2024 2024-09-14T08:53:16.586 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:53:16 2024 2024-09-14T08:53:16.586 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:53:16.586 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:53:16.587 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: dc8ae139-d4cc-4aab-82f3-d300cef64dfd 2024-09-14T08:53:16.587 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: true 2024-09-14T08:53:16.589 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n '' ']' 2024-09-14T08:53:16.589 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ stress_write_image cluster2 mirror image_50 2024-09-14T08:53:16.589 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:53:16.589 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:53:16.589 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_50 2024-09-14T08:53:16.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T08:53:16.591 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local duration=9 2024-09-14T08:53:16.591 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set +e 2024-09-14T08:53:16.591 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ timeout 9s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_50 --debug-rbd=20 --debug-journaler=20 2024-09-14T08:53:16.735 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:100 IOs, 14475264 bytes 2024-09-14T08:53:16.835 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:200 IOs, 15076352 bytes 2024-09-14T08:53:17.041 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:300 IOs, 69366784 bytes 2024-09-14T08:53:17.245 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:400 IOs, 125432832 bytes 2024-09-14T08:53:17.435 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:500 IOs, 166430720 bytes 2024-09-14T08:53:17.639 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:600 IOs, 214392832 bytes 2024-09-14T08:53:17.857 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:700 IOs, 264853504 bytes 2024-09-14T08:53:18.025 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:800 IOs, 298398720 bytes 2024-09-14T08:53:18.247 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:900 IOs, 358908928 bytes 2024-09-14T08:53:18.409 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1000 IOs, 388268032 bytes 2024-09-14T08:53:18.598 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1100 IOs, 443039744 bytes 2024-09-14T08:53:18.796 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1200 IOs, 479493120 bytes 2024-09-14T08:53:18.910 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1300 IOs, 491930624 bytes 2024-09-14T08:53:19.028 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1400 IOs, 511684608 bytes 2024-09-14T08:53:19.285 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1500 IOs, 570509312 bytes 2024-09-14T08:53:19.488 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1600 IOs, 635162624 bytes 2024-09-14T08:53:19.797 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1700 IOs, 704730112 bytes 2024-09-14T08:53:19.968 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1800 IOs, 741292032 bytes 2024-09-14T08:53:20.100 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1900 IOs, 759094272 bytes 2024-09-14T08:53:20.197 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2000 IOs, 769250304 bytes 2024-09-14T08:53:20.375 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2100 IOs, 806581248 bytes 2024-09-14T08:53:20.469 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2200 IOs, 807202816 bytes 2024-09-14T08:53:20.611 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2300 IOs, 829888512 bytes 2024-09-14T08:53:20.793 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2400 IOs, 879483904 bytes 2024-09-14T08:53:20.957 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2500 IOs, 904403968 bytes 2024-09-14T08:53:21.081 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2600 IOs, 928428032 bytes 2024-09-14T08:53:21.210 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2700 IOs, 940786688 bytes 2024-09-14T08:53:21.397 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2800 IOs, 984926208 bytes 2024-09-14T08:53:21.563 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2900 IOs, 1016009728 bytes 2024-09-14T08:53:21.771 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3000 IOs, 1086441472 bytes 2024-09-14T08:53:21.965 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3100 IOs, 1128001536 bytes 2024-09-14T08:53:22.142 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3200 IOs, 1145782272 bytes 2024-09-14T08:53:22.449 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3300 IOs, 1235968000 bytes 2024-09-14T08:53:22.607 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3400 IOs, 1286905856 bytes 2024-09-14T08:53:22.829 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3500 IOs, 1324459008 bytes 2024-09-14T08:53:22.999 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3600 IOs, 1357537280 bytes 2024-09-14T08:53:23.221 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3700 IOs, 1406873600 bytes 2024-09-14T08:53:23.365 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3800 IOs, 1431221248 bytes 2024-09-14T08:53:23.635 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3900 IOs, 1504190464 bytes 2024-09-14T08:53:23.875 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4000 IOs, 1576401920 bytes 2024-09-14T08:53:24.210 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4100 IOs, 1673207808 bytes 2024-09-14T08:53:24.432 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4200 IOs, 1731415040 bytes 2024-09-14T08:53:24.674 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4300 IOs, 1797175296 bytes 2024-09-14T08:53:24.891 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4400 IOs, 1841873920 bytes 2024-09-14T08:53:25.037 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4500 IOs, 1865141248 bytes 2024-09-14T08:53:25.186 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4600 IOs, 1893880832 bytes 2024-09-14T08:53:25.442 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4700 IOs, 1962098688 bytes 2024-09-14T08:53:25.598 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ error_code=124 2024-09-14T08:53:25.599 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set -e 2024-09-14T08:53:25.599 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T08:53:25.599 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:53:25.599 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_pool_images cluster2 mirror 50 2024-09-14T08:53:25.599 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:53:25.599 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:53:25.599 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_count=50 2024-09-14T08:53:25.599 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:53:25.599 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local count 2024-09-14T08:53:25.599 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local last_count=0 2024-09-14T08:53:25.600 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:53:25.600 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ seq 1 40 2024-09-14T08:53:25.600 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in `seq 1 40` 2024-09-14T08:53:25.600 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 1 -ne 1 2024-09-14T08:53:25.601 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster cluster2 -p mirror mirror pool status 2024-09-14T08:53:25.601 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ grep 'images: ' 2024-09-14T08:53:25.601 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cut '-d ' -f 2 2024-09-14T08:53:25.640 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: failed to query services: (13) Permission denied 2024-09-14T08:53:25.644 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ count=50 2024-09-14T08:53:25.644 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 50 = 50 2024-09-14T08:53:25.644 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:53:25.644 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_pool_healthy cluster2 mirror 2024-09-14T08:53:25.644 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:53:25.644 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:53:25.644 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:53:25.644 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state 2024-09-14T08:53:25.644 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ seq 1 40 2024-09-14T08:53:25.644 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in `seq 1 40` 2024-09-14T08:53:25.645 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 1 -ne 1 2024-09-14T08:53:25.645 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster cluster2 -p mirror mirror pool status 2024-09-14T08:53:25.645 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ grep 'image health:' 2024-09-14T08:53:25.645 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cut '-d ' -f 3 2024-09-14T08:53:25.682 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: failed to query services: (13) Permission denied 2024-09-14T08:53:25.686 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ state=OK 2024-09-14T08:53:25.686 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test OK = ERROR 2024-09-14T08:53:25.687 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test OK = OK 2024-09-14T08:53:25.687 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:53:25.687 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_pool_images cluster1 mirror 50 2024-09-14T08:53:25.687 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:53:25.687 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:53:25.687 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_count=50 2024-09-14T08:53:25.687 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:53:25.687 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local count 2024-09-14T08:53:25.687 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local last_count=0 2024-09-14T08:53:25.687 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:53:25.687 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ seq 1 40 2024-09-14T08:53:25.687 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in `seq 1 40` 2024-09-14T08:53:25.688 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 1 -ne 1 2024-09-14T08:53:25.688 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster cluster1 -p mirror mirror pool status 2024-09-14T08:53:25.688 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ grep 'images: ' 2024-09-14T08:53:25.688 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cut '-d ' -f 2 2024-09-14T08:53:25.727 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: failed to query services: (13) Permission denied 2024-09-14T08:53:25.730 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ count=50 2024-09-14T08:53:25.730 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 50 = 50 2024-09-14T08:53:25.730 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:53:25.730 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_pool_healthy cluster1 mirror 2024-09-14T08:53:25.730 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:53:25.730 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:53:25.731 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:53:25.731 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state 2024-09-14T08:53:25.731 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ seq 1 40 2024-09-14T08:53:25.731 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in `seq 1 40` 2024-09-14T08:53:25.731 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 1 -ne 1 2024-09-14T08:53:25.732 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster cluster1 -p mirror mirror pool status 2024-09-14T08:53:25.732 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ grep 'image health:' 2024-09-14T08:53:25.732 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cut '-d ' -f 3 2024-09-14T08:53:25.771 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: failed to query services: (13) Permission denied 2024-09-14T08:53:25.774 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ state=WARNING 2024-09-14T08:53:25.774 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test WARNING = ERROR 2024-09-14T08:53:25.774 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test WARNING = OK 2024-09-14T08:53:25.774 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in `seq 1 40` 2024-09-14T08:53:25.774 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 2 -ne 1 2024-09-14T08:53:25.775 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 30 2024-09-14T08:53:55.776 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster cluster1 -p mirror mirror pool status 2024-09-14T08:53:55.777 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ grep 'image health:' 2024-09-14T08:53:55.777 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cut '-d ' -f 3 2024-09-14T08:53:55.822 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: failed to query services: (13) Permission denied 2024-09-14T08:53:55.826 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ state=OK 2024-09-14T08:53:55.826 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test OK = ERROR 2024-09-14T08:53:55.827 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test OK = OK 2024-09-14T08:53:55.827 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:53:55.827 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ testlog 'TEST: compare many images' 2024-09-14T08:53:55.827 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ tee -a /tmp/tmp.IUd9LESfOA/rbd-mirror.test.log 2024-09-14T08:53:55.827 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ date '+%F %T' 2024-09-14T08:53:55.827 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ echo 2024-09-14 08:53:55 TEST: compare many images 2024-09-14T08:53:55.827 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:2024-09-14 08:53:55 TEST: compare many images 2024-09-14T08:53:55.828 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ seq 1 50 2024-09-14T08:53:55.829 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T08:53:55.829 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_1 2024-09-14T08:53:55.829 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_snap cluster2 mirror image_1 snap 2024-09-14T08:53:55.830 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:53:55.830 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:53:55.830 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_1 2024-09-14T08:53:55.830 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:53:55.830 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror snap create image_1@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T08:53:56.583 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_started cluster1 mirror image_1 2024-09-14T08:53:56.583 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:53:56.583 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:53:56.583 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_1 2024-09-14T08:53:56.583 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_state cluster1 mirror image_1 started 2024-09-14T08:53:56.583 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:53:56.583 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:53:56.583 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_1 2024-09-14T08:53:56.583 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=started 2024-09-14T08:53:56.583 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:53:56.583 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:53:56.583 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:53:57.584 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_replay_state cluster1 mirror image_1 started 2024-09-14T08:53:57.584 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:53:57.584 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:53:57.585 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_1 2024-09-14T08:53:57.585 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=started 2024-09-14T08:53:57.585 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_result 2024-09-14T08:53:57.585 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=stopped 2024-09-14T08:53:57.586 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_1 2024-09-14T08:53:57.586 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ grep -i state 2024-09-14T08:53:57.586 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_instance=cluster1 2024-09-14T08:53:57.586 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:53:57.586 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T08:53:57.586 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance=cluster1 2024-09-14T08:53:57.587 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local loop_instance 2024-09-14T08:53:57.587 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:53:57.587 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sleep 0 2024-09-14T08:53:57.587 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T08:53:57.587 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ seq 0 3 2024-09-14T08:53:57.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:53:57.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_1 2024-09-14T08:53:57.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:0 2024-09-14T08:53:57.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:53:57.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:53:57.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T08:53:57.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:0 2024-09-14T08:53:57.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:53:57.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:53:57.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:53:57.591 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=0 2024-09-14T08:53:57.591 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T08:53:57.591 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:53:57.591 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:53:57.591 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=0 2024-09-14T08:53:57.591 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:53:57.591 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T08:53:57.591 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:0 2024-09-14T08:53:57.591 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:53:57.591 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:53:57.591 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T08:53:57.591 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:0 2024-09-14T08:53:57.592 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:53:57.592 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:53:57.592 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:53:57.592 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:53:57.592 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T08:53:57.592 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:53:57.592 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:53:57.592 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=0 2024-09-14T08:53:57.592 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=0 2024-09-14T08:53:57.592 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T08:53:57.611 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:53:57.612 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:53:57.612 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:53:57.612 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_1 2024-09-14T08:53:57.700 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:no valid command found; 2 closest matches: 2024-09-14T08:53:57.700 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status mirror/image_12 2024-09-14T08:53:57.700 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status mirror/image_19 2024-09-14T08:53:57.700 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:53:57.703 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:53:57.703 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_1 2024-09-14T08:53:57.703 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:1 2024-09-14T08:53:57.703 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:53:57.703 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:53:57.704 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T08:53:57.704 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:1 2024-09-14T08:53:57.704 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:53:57.704 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:53:57.704 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:53:57.704 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=1 2024-09-14T08:53:57.704 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T08:53:57.704 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:53:57.704 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:53:57.704 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=1 2024-09-14T08:53:57.704 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:53:57.704 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T08:53:57.704 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:1 2024-09-14T08:53:57.704 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:53:57.704 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:53:57.705 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T08:53:57.705 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:1 2024-09-14T08:53:57.705 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:53:57.705 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:53:57.705 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:53:57.705 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:53:57.705 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T08:53:57.705 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:53:57.705 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:53:57.705 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=1 2024-09-14T08:53:57.705 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=1 2024-09-14T08:53:57.705 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T08:53:57.722 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:53:57.722 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:53:57.722 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:53:57.722 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_1 2024-09-14T08:53:57.802 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:no valid command found; 2 closest matches: 2024-09-14T08:53:57.803 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status mirror/image_10 2024-09-14T08:53:57.803 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status mirror/image_16 2024-09-14T08:53:57.803 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:53:57.806 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:53:57.806 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_1 2024-09-14T08:53:57.806 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:2 2024-09-14T08:53:57.806 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:53:57.806 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:53:57.806 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-14T08:53:57.806 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:2 2024-09-14T08:53:57.806 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:53:57.806 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:53:57.807 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:53:57.807 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=2 2024-09-14T08:53:57.807 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T08:53:57.807 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:53:57.807 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:53:57.807 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=2 2024-09-14T08:53:57.807 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:53:57.807 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-14T08:53:57.807 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:2 2024-09-14T08:53:57.807 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:53:57.807 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:53:57.807 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T08:53:57.807 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:2 2024-09-14T08:53:57.807 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:53:57.808 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:53:57.808 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:53:57.808 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:53:57.808 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-14T08:53:57.808 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:53:57.808 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:53:57.808 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=2 2024-09-14T08:53:57.808 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=2 2024-09-14T08:53:57.808 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T08:53:57.825 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:53:57.825 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:53:57.825 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:53:57.825 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_1 2024-09-14T08:53:57.913 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ return 0 2024-09-14T08:53:57.913 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T08:53:57.914 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ echo ' "state": "Replaying"' 2024-09-14T08:53:57.914 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep -i Replaying 2024-09-14T08:53:57.914 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: "state": "Replaying" 2024-09-14T08:53:57.916 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_state=started 2024-09-14T08:53:57.916 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test started = started 2024-09-14T08:53:57.916 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:53:57.916 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_1 2024-09-14T08:53:57.917 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:53:57.917 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:53:57.917 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:53:57.917 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_1 2024-09-14T08:53:57.917 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = journal ']' 2024-09-14T08:53:57.917 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = snapshot ']' 2024-09-14T08:53:57.917 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_1 2024-09-14T08:53:57.917 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:53:57.917 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:53:57.917 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:53:57.917 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_1 2024-09-14T08:53:57.917 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster2-mirror-image_1.status 2024-09-14T08:53:57.917 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster2-mirror-image_1.status 2024-09-14T08:53:57.918 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:53:57.918 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_1.status 2024-09-14T08:53:57.918 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster1-mirror-image_1.status 2024-09-14T08:53:57.919 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster1-mirror-image_1.status 2024-09-14T08:53:57.919 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:53:57.919 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_status_log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_1.status 2024-09-14T08:53:57.919 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ mirror_image_snapshot cluster2 mirror image_1 2024-09-14T08:53:57.919 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:53:57.919 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:53:57.919 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_1 2024-09-14T08:53:57.919 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_1 2024-09-14T08:53:58.598 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Snapshot ID: 86 2024-09-14T08:53:58.605 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_1 /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_1.status 2024-09-14T08:53:58.605 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:53:58.605 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:53:58.605 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_1 2024-09-14T08:53:58.605 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_1.status 2024-09-14T08:53:58.605 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_1 --format xml 2024-09-14T08:53:58.605 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:53:58.650 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-14T08:53:58.652 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snapshot_id=86 2024-09-14T08:53:58.652 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:53:58.652 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:53:58.653 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.2 2024-09-14T08:53:58.853 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_1 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_1.status 2024-09-14T08:53:58.854 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:53:58.854 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:53:58.854 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_1 2024-09-14T08:53:58.854 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_1.status 2024-09-14T08:53:58.854 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_1 --format xml 2024-09-14T08:53:58.854 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:53:58.903 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:53:58.905 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=35 2024-09-14T08:53:58.905 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 86 = 35 2024-09-14T08:53:58.905 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:53:58.905 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.4 2024-09-14T08:53:59.306 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_1 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_1.status 2024-09-14T08:53:59.306 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:53:59.307 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:53:59.307 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_1 2024-09-14T08:53:59.307 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_1.status 2024-09-14T08:53:59.307 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_1 --format xml 2024-09-14T08:53:59.307 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:53:59.357 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:53:59.359 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=35 2024-09-14T08:53:59.359 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 86 = 35 2024-09-14T08:53:59.359 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:53:59.359 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.8 2024-09-14T08:54:00.160 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_1 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_1.status 2024-09-14T08:54:00.160 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:54:00.160 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:00.160 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_1 2024-09-14T08:54:00.160 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_1.status 2024-09-14T08:54:00.160 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_1 --format xml 2024-09-14T08:54:00.161 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:54:00.215 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:54:00.215 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:54:00.217 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:54:00.217 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:54:00.217 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:54:00.217 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:54:00.217 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 86 = '' 2024-09-14T08:54:00.217 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:54:00.217 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1.6 2024-09-14T08:54:01.819 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_1 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_1.status 2024-09-14T08:54:01.819 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:54:01.819 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:01.819 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_1 2024-09-14T08:54:01.819 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_1.status 2024-09-14T08:54:01.819 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_1 --format xml 2024-09-14T08:54:01.819 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:54:01.873 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:54:01.876 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=86 2024-09-14T08:54:01.876 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 86 = 86 2024-09-14T08:54:01.876 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:54:01.876 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snap_present cluster1 mirror image_1 snap 2024-09-14T08:54:01.876 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:54:01.876 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:01.876 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_1 2024-09-14T08:54:01.876 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:54:01.876 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:54:01.876 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T08:54:01.876 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:54:02.877 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 info mirror/image_1@snap 2024-09-14T08:54:02.924 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_1': 2024-09-14T08:54:02.924 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T08:54:02.924 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:54:02.924 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 2 2024-09-14T08:54:02.924 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 10bdc613bf78 2024-09-14T08:54:02.924 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.10bdc613bf78 2024-09-14T08:54:02.925 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:54:02.925 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering, non-primary 2024-09-14T08:54:02.925 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:54:02.925 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:54:02.925 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:45:22 2024 2024-09-14T08:54:02.925 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:45:22 2024 2024-09-14T08:54:02.925 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:45:22 2024 2024-09-14T08:54:02.925 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: protected: False 2024-09-14T08:54:02.925 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:54:02.925 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:54:02.925 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: 06bb8003-76f2-47e8-a72f-feadf752d8d3 2024-09-14T08:54:02.925 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: false 2024-09-14T08:54:02.928 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:54:02.928 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ compare_image_snaps mirror image_1 snap 2024-09-14T08:54:02.928 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:02.929 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_1 2024-09-14T08:54:02.929 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:54:02.929 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local ret=0 2024-09-14T08:54:02.929 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local rmt_export=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_1.export 2024-09-14T08:54:02.929 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local loc_export=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_1.export 2024-09-14T08:54:02.929 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_1.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_1.export 2024-09-14T08:54:02.929 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror export image_1@snap /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_1.export 2024-09-14T08:54:03.130 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T08:54:03.137 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 -p mirror export image_1@snap /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_1.export 2024-09-14T08:54:03.375 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T08:54:03.381 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cmp /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_1.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_1.export 2024-09-14T08:54:03.436 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_1.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_1.export 2024-09-14T08:54:03.468 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:54:03.468 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T08:54:03.468 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_2 2024-09-14T08:54:03.468 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_snap cluster2 mirror image_2 snap 2024-09-14T08:54:03.469 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:54:03.469 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:03.469 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_2 2024-09-14T08:54:03.469 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:54:03.469 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror snap create image_2@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T08:54:03.604 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_started cluster1 mirror image_2 2024-09-14T08:54:03.604 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:54:03.605 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:03.605 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_2 2024-09-14T08:54:03.605 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_state cluster1 mirror image_2 started 2024-09-14T08:54:03.605 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:54:03.605 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:03.605 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_2 2024-09-14T08:54:03.605 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=started 2024-09-14T08:54:03.605 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:54:03.605 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:54:03.605 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:54:04.605 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_replay_state cluster1 mirror image_2 started 2024-09-14T08:54:04.606 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:54:04.606 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:04.606 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_2 2024-09-14T08:54:04.606 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=started 2024-09-14T08:54:04.606 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_result 2024-09-14T08:54:04.606 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=stopped 2024-09-14T08:54:04.607 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_2 2024-09-14T08:54:04.607 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_instance=cluster1 2024-09-14T08:54:04.607 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:54:04.607 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ grep -i state 2024-09-14T08:54:04.607 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T08:54:04.608 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance=cluster1 2024-09-14T08:54:04.608 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local loop_instance 2024-09-14T08:54:04.608 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:54:04.608 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sleep 0 2024-09-14T08:54:04.608 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T08:54:04.608 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ seq 0 3 2024-09-14T08:54:04.611 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:54:04.611 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_2 2024-09-14T08:54:04.611 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:0 2024-09-14T08:54:04.611 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:54:04.611 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:54:04.611 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T08:54:04.611 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:0 2024-09-14T08:54:04.611 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:54:04.611 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:54:04.611 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:54:04.611 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=0 2024-09-14T08:54:04.612 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T08:54:04.612 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:54:04.612 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:54:04.612 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=0 2024-09-14T08:54:04.612 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:54:04.612 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T08:54:04.612 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:0 2024-09-14T08:54:04.612 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:54:04.612 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:54:04.612 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T08:54:04.612 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:0 2024-09-14T08:54:04.612 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:54:04.612 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:54:04.613 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:54:04.613 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:54:04.613 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T08:54:04.613 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:54:04.613 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:54:04.613 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=0 2024-09-14T08:54:04.613 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=0 2024-09-14T08:54:04.613 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T08:54:04.632 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:54:04.632 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:54:04.632 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:54:04.632 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_2 2024-09-14T08:54:04.728 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ return 0 2024-09-14T08:54:04.729 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T08:54:04.729 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ echo ' "state": "Replaying"' 2024-09-14T08:54:04.729 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep -i Replaying 2024-09-14T08:54:04.730 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: "state": "Replaying" 2024-09-14T08:54:04.731 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_state=started 2024-09-14T08:54:04.731 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test started = started 2024-09-14T08:54:04.731 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:54:04.731 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_2 2024-09-14T08:54:04.732 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:54:04.732 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:54:04.732 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:04.732 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_2 2024-09-14T08:54:04.732 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = journal ']' 2024-09-14T08:54:04.732 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = snapshot ']' 2024-09-14T08:54:04.732 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_2 2024-09-14T08:54:04.732 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:54:04.732 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:54:04.732 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:04.732 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_2 2024-09-14T08:54:04.732 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster2-mirror-image_2.status 2024-09-14T08:54:04.732 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster2-mirror-image_2.status 2024-09-14T08:54:04.733 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:54:04.734 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_2.status 2024-09-14T08:54:04.734 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster1-mirror-image_2.status 2024-09-14T08:54:04.734 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster1-mirror-image_2.status 2024-09-14T08:54:04.734 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:54:04.734 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_status_log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_2.status 2024-09-14T08:54:04.734 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ mirror_image_snapshot cluster2 mirror image_2 2024-09-14T08:54:04.734 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:54:04.734 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:04.734 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_2 2024-09-14T08:54:04.734 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_2 2024-09-14T08:54:05.618 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Snapshot ID: 88 2024-09-14T08:54:05.624 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_2 /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_2.status 2024-09-14T08:54:05.625 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:54:05.625 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:05.625 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_2 2024-09-14T08:54:05.625 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_2.status 2024-09-14T08:54:05.625 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_2 --format xml 2024-09-14T08:54:05.625 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:54:05.671 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-14T08:54:05.673 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snapshot_id=88 2024-09-14T08:54:05.673 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:54:05.673 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:54:05.673 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.2 2024-09-14T08:54:05.875 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_2 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_2.status 2024-09-14T08:54:05.875 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:54:05.875 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:05.875 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_2 2024-09-14T08:54:05.875 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_2.status 2024-09-14T08:54:05.875 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_2 --format xml 2024-09-14T08:54:05.875 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:54:05.925 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:54:05.927 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=36 2024-09-14T08:54:05.927 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 88 = 36 2024-09-14T08:54:05.927 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:54:05.927 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.4 2024-09-14T08:54:06.328 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_2 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_2.status 2024-09-14T08:54:06.328 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:54:06.329 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:06.329 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_2 2024-09-14T08:54:06.329 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_2.status 2024-09-14T08:54:06.329 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_2 --format xml 2024-09-14T08:54:06.329 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:54:06.392 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:54:06.392 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:54:06.394 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:54:06.394 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:54:06.394 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:54:06.394 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:54:06.394 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 88 = '' 2024-09-14T08:54:06.394 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:54:06.394 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.8 2024-09-14T08:54:07.195 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_2 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_2.status 2024-09-14T08:54:07.196 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:54:07.196 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:07.196 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_2 2024-09-14T08:54:07.196 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_2.status 2024-09-14T08:54:07.196 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_2 --format xml 2024-09-14T08:54:07.196 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:54:07.250 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:54:07.251 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:54:07.252 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:54:07.252 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:54:07.252 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:54:07.253 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:54:07.253 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 88 = '' 2024-09-14T08:54:07.253 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:54:07.253 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1.6 2024-09-14T08:54:08.854 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_2 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_2.status 2024-09-14T08:54:08.854 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:54:08.854 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:08.854 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_2 2024-09-14T08:54:08.854 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_2.status 2024-09-14T08:54:08.855 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_2 --format xml 2024-09-14T08:54:08.855 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:54:08.911 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:54:08.913 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=88 2024-09-14T08:54:08.913 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 88 = 88 2024-09-14T08:54:08.913 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:54:08.913 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snap_present cluster1 mirror image_2 snap 2024-09-14T08:54:08.914 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:54:08.914 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:08.914 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_2 2024-09-14T08:54:08.914 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:54:08.914 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:54:08.914 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T08:54:08.914 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:54:09.914 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 info mirror/image_2@snap 2024-09-14T08:54:09.961 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_2': 2024-09-14T08:54:09.961 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T08:54:09.961 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:54:09.961 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 2 2024-09-14T08:54:09.961 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 10c2e31c73db 2024-09-14T08:54:09.961 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.10c2e31c73db 2024-09-14T08:54:09.961 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:54:09.961 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering, non-primary 2024-09-14T08:54:09.962 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:54:09.962 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:54:09.962 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:45:31 2024 2024-09-14T08:54:09.962 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:45:31 2024 2024-09-14T08:54:09.962 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:45:31 2024 2024-09-14T08:54:09.962 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: protected: False 2024-09-14T08:54:09.962 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:54:09.962 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:54:09.962 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: f36140bd-57e9-420f-8e29-5672a8b58d3d 2024-09-14T08:54:09.962 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: false 2024-09-14T08:54:09.965 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:54:09.966 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ compare_image_snaps mirror image_2 snap 2024-09-14T08:54:09.966 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:09.966 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_2 2024-09-14T08:54:09.966 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:54:09.966 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local ret=0 2024-09-14T08:54:09.966 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local rmt_export=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_2.export 2024-09-14T08:54:09.966 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local loc_export=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_2.export 2024-09-14T08:54:09.966 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_2.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_2.export 2024-09-14T08:54:09.966 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror export image_2@snap /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_2.export 2024-09-14T08:54:10.156 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T08:54:10.164 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 -p mirror export image_2@snap /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_2.export 2024-09-14T08:54:10.405 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T08:54:10.412 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cmp /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_2.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_2.export 2024-09-14T08:54:10.467 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_2.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_2.export 2024-09-14T08:54:10.497 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:54:10.497 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T08:54:10.498 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_3 2024-09-14T08:54:10.498 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_snap cluster2 mirror image_3 snap 2024-09-14T08:54:10.498 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:54:10.498 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:10.498 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_3 2024-09-14T08:54:10.498 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:54:10.498 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror snap create image_3@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T08:54:10.613 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_started cluster1 mirror image_3 2024-09-14T08:54:10.613 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:54:10.613 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:10.613 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_3 2024-09-14T08:54:10.613 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_state cluster1 mirror image_3 started 2024-09-14T08:54:10.613 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:54:10.613 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:10.613 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_3 2024-09-14T08:54:10.613 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=started 2024-09-14T08:54:10.613 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:54:10.613 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:54:10.614 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:54:11.614 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_replay_state cluster1 mirror image_3 started 2024-09-14T08:54:11.614 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:54:11.614 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:11.614 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_3 2024-09-14T08:54:11.614 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=started 2024-09-14T08:54:11.614 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_result 2024-09-14T08:54:11.614 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=stopped 2024-09-14T08:54:11.614 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_3 2024-09-14T08:54:11.616 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_instance=cluster1 2024-09-14T08:54:11.616 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:54:11.616 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ grep -i state 2024-09-14T08:54:11.616 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T08:54:11.616 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance=cluster1 2024-09-14T08:54:11.616 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local loop_instance 2024-09-14T08:54:11.616 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:54:11.616 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sleep 0 2024-09-14T08:54:11.616 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T08:54:11.616 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ seq 0 3 2024-09-14T08:54:11.619 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:54:11.619 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_3 2024-09-14T08:54:11.619 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:0 2024-09-14T08:54:11.619 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:54:11.619 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:54:11.619 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T08:54:11.619 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:0 2024-09-14T08:54:11.619 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:54:11.619 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:54:11.620 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:54:11.620 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=0 2024-09-14T08:54:11.620 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T08:54:11.620 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:54:11.620 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:54:11.620 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=0 2024-09-14T08:54:11.620 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:54:11.620 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T08:54:11.620 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:0 2024-09-14T08:54:11.620 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:54:11.620 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:54:11.621 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T08:54:11.621 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:0 2024-09-14T08:54:11.621 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:54:11.621 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:54:11.621 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:54:11.621 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:54:11.621 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T08:54:11.621 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:54:11.621 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:54:11.621 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=0 2024-09-14T08:54:11.621 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=0 2024-09-14T08:54:11.621 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T08:54:11.640 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:54:11.640 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:54:11.640 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:54:11.640 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_3 2024-09-14T08:54:11.728 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:no valid command found; 3 closest matches: 2024-09-14T08:54:11.728 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status mirror/image_31 2024-09-14T08:54:11.728 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status mirror/image_32 2024-09-14T08:54:11.728 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status mirror/image_36 2024-09-14T08:54:11.728 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:54:11.731 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:54:11.731 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_3 2024-09-14T08:54:11.731 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:1 2024-09-14T08:54:11.731 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:54:11.731 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:54:11.731 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T08:54:11.731 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:1 2024-09-14T08:54:11.731 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:54:11.731 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:54:11.731 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:54:11.732 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=1 2024-09-14T08:54:11.732 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T08:54:11.732 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:54:11.732 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:54:11.732 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=1 2024-09-14T08:54:11.732 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:54:11.732 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T08:54:11.732 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:1 2024-09-14T08:54:11.732 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:54:11.732 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:54:11.732 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T08:54:11.732 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:1 2024-09-14T08:54:11.732 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:54:11.732 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:54:11.732 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:54:11.733 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:54:11.733 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T08:54:11.733 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:54:11.733 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:54:11.733 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=1 2024-09-14T08:54:11.733 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=1 2024-09-14T08:54:11.733 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T08:54:11.751 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:54:11.751 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:54:11.751 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:54:11.751 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_3 2024-09-14T08:54:11.833 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:no valid command found; 2 closest matches: 2024-09-14T08:54:11.833 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status mirror/image_35 2024-09-14T08:54:11.833 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status mirror/image_39 2024-09-14T08:54:11.833 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:54:11.836 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:54:11.836 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_3 2024-09-14T08:54:11.836 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:2 2024-09-14T08:54:11.836 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:54:11.836 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:54:11.836 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-14T08:54:11.837 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:2 2024-09-14T08:54:11.837 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:54:11.837 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:54:11.837 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:54:11.837 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=2 2024-09-14T08:54:11.837 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T08:54:11.837 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:54:11.837 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:54:11.837 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=2 2024-09-14T08:54:11.837 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:54:11.837 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-14T08:54:11.837 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:2 2024-09-14T08:54:11.837 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:54:11.837 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:54:11.838 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T08:54:11.838 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:2 2024-09-14T08:54:11.838 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:54:11.838 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:54:11.838 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:54:11.838 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:54:11.838 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-14T08:54:11.838 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:54:11.838 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:54:11.838 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=2 2024-09-14T08:54:11.838 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=2 2024-09-14T08:54:11.839 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T08:54:11.854 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:54:11.854 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:54:11.855 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:54:11.855 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_3 2024-09-14T08:54:11.937 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:no valid command found; 2 closest matches: 2024-09-14T08:54:11.938 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status mirror/image_33 2024-09-14T08:54:11.938 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status mirror/image_37 2024-09-14T08:54:11.938 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:54:11.942 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:54:11.942 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_3 2024-09-14T08:54:11.942 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:3 2024-09-14T08:54:11.942 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:54:11.942 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:54:11.942 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:3 cluster instance 2024-09-14T08:54:11.942 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:3 2024-09-14T08:54:11.942 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:54:11.942 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:54:11.942 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:54:11.942 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=3 2024-09-14T08:54:11.943 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-14T08:54:11.943 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:54:11.943 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:54:11.943 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=3 2024-09-14T08:54:11.943 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=3 2024-09-14T08:54:11.943 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:3 cluster1 2024-09-14T08:54:11.943 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:3 2024-09-14T08:54:11.943 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:54:11.943 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:54:11.944 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-14T08:54:11.944 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:3 2024-09-14T08:54:11.944 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:54:11.944 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:54:11.944 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:54:11.944 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=3 2024-09-14T08:54:11.944 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 3 = cluster1:3 ']' 2024-09-14T08:54:11.944 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:54:11.944 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:54:11.944 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=3 2024-09-14T08:54:11.944 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=3 2024-09-14T08:54:11.944 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-14T08:54:11.959 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T08:54:11.959 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T08:54:11.959 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T08:54:11.959 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_3 2024-09-14T08:54:12.054 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ return 0 2024-09-14T08:54:12.055 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T08:54:12.055 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ echo ' "state": "Replaying"' 2024-09-14T08:54:12.055 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep -i Replaying 2024-09-14T08:54:12.055 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: "state": "Replaying" 2024-09-14T08:54:12.057 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_state=started 2024-09-14T08:54:12.057 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test started = started 2024-09-14T08:54:12.057 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:54:12.057 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_3 2024-09-14T08:54:12.057 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:54:12.057 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:54:12.057 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:12.057 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_3 2024-09-14T08:54:12.057 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = journal ']' 2024-09-14T08:54:12.058 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = snapshot ']' 2024-09-14T08:54:12.058 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_3 2024-09-14T08:54:12.058 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:54:12.058 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:54:12.058 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:12.058 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_3 2024-09-14T08:54:12.058 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster2-mirror-image_3.status 2024-09-14T08:54:12.058 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster2-mirror-image_3.status 2024-09-14T08:54:12.058 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:54:12.059 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_3.status 2024-09-14T08:54:12.060 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster1-mirror-image_3.status 2024-09-14T08:54:12.060 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster1-mirror-image_3.status 2024-09-14T08:54:12.060 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:54:12.060 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_status_log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_3.status 2024-09-14T08:54:12.060 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ mirror_image_snapshot cluster2 mirror image_3 2024-09-14T08:54:12.060 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:54:12.060 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:12.060 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_3 2024-09-14T08:54:12.060 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_3 2024-09-14T08:54:12.623 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Snapshot ID: 90 2024-09-14T08:54:12.630 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_3 /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_3.status 2024-09-14T08:54:12.630 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:54:12.630 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:12.630 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_3 2024-09-14T08:54:12.630 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_3.status 2024-09-14T08:54:12.630 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_3 --format xml 2024-09-14T08:54:12.631 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:54:12.678 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-14T08:54:12.680 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snapshot_id=90 2024-09-14T08:54:12.680 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:54:12.680 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:54:12.680 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.2 2024-09-14T08:54:12.881 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_3 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_3.status 2024-09-14T08:54:12.881 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:54:12.881 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:12.881 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_3 2024-09-14T08:54:12.881 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_3.status 2024-09-14T08:54:12.881 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_3 --format xml 2024-09-14T08:54:12.882 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:54:12.930 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:54:12.931 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=37 2024-09-14T08:54:12.932 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 90 = 37 2024-09-14T08:54:12.932 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:54:12.932 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.4 2024-09-14T08:54:13.333 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_3 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_3.status 2024-09-14T08:54:13.333 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:54:13.333 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:13.333 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_3 2024-09-14T08:54:13.333 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_3.status 2024-09-14T08:54:13.334 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_3 --format xml 2024-09-14T08:54:13.334 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:54:13.382 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:54:13.384 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=37 2024-09-14T08:54:13.384 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 90 = 37 2024-09-14T08:54:13.384 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:54:13.384 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.8 2024-09-14T08:54:14.185 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_3 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_3.status 2024-09-14T08:54:14.186 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:54:14.186 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:14.186 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_3 2024-09-14T08:54:14.186 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_3.status 2024-09-14T08:54:14.186 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_3 --format xml 2024-09-14T08:54:14.186 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:54:14.239 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:54:14.240 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:54:14.241 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:54:14.241 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:54:14.241 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:54:14.242 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:54:14.242 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 90 = '' 2024-09-14T08:54:14.242 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:54:14.242 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1.6 2024-09-14T08:54:15.843 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_3 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_3.status 2024-09-14T08:54:15.843 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:54:15.843 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:15.843 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_3 2024-09-14T08:54:15.843 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_3.status 2024-09-14T08:54:15.844 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_3 --format xml 2024-09-14T08:54:15.844 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:54:15.905 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:54:15.907 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=90 2024-09-14T08:54:15.907 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 90 = 90 2024-09-14T08:54:15.907 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:54:15.907 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snap_present cluster1 mirror image_3 snap 2024-09-14T08:54:15.907 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:54:15.907 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:15.907 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_3 2024-09-14T08:54:15.908 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:54:15.908 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:54:15.908 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T08:54:15.908 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:54:16.908 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 info mirror/image_3@snap 2024-09-14T08:54:16.955 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_3': 2024-09-14T08:54:16.956 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T08:54:16.956 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:54:16.956 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 2 2024-09-14T08:54:16.956 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 11898f7e4a34 2024-09-14T08:54:16.956 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.11898f7e4a34 2024-09-14T08:54:16.956 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:54:16.956 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering, non-primary 2024-09-14T08:54:16.956 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:54:16.956 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:54:16.956 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:45:38 2024 2024-09-14T08:54:16.956 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:45:38 2024 2024-09-14T08:54:16.957 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:45:38 2024 2024-09-14T08:54:16.957 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: protected: False 2024-09-14T08:54:16.957 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:54:16.957 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:54:16.957 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: c4fe1bd8-3565-46e8-87a1-574d1ec9a443 2024-09-14T08:54:16.957 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: false 2024-09-14T08:54:16.960 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:54:16.960 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ compare_image_snaps mirror image_3 snap 2024-09-14T08:54:16.960 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:16.960 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_3 2024-09-14T08:54:16.960 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:54:16.960 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local ret=0 2024-09-14T08:54:16.960 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local rmt_export=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_3.export 2024-09-14T08:54:16.960 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local loc_export=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_3.export 2024-09-14T08:54:16.960 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_3.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_3.export 2024-09-14T08:54:16.961 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror export image_3@snap /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_3.export 2024-09-14T08:54:17.363 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T08:54:17.371 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 -p mirror export image_3@snap /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_3.export 2024-09-14T08:54:17.630 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T08:54:17.637 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cmp /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_3.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_3.export 2024-09-14T08:54:17.693 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_3.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_3.export 2024-09-14T08:54:17.724 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:54:17.724 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T08:54:17.725 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_4 2024-09-14T08:54:17.725 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_snap cluster2 mirror image_4 snap 2024-09-14T08:54:17.725 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:54:17.725 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:17.725 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_4 2024-09-14T08:54:17.725 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:54:17.725 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror snap create image_4@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T08:54:18.631 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_started cluster1 mirror image_4 2024-09-14T08:54:18.631 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:54:18.632 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:18.632 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_4 2024-09-14T08:54:18.632 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_state cluster1 mirror image_4 started 2024-09-14T08:54:18.632 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:54:18.632 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:18.632 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_4 2024-09-14T08:54:18.632 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=started 2024-09-14T08:54:18.632 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:54:18.632 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:54:18.632 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:54:19.633 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_replay_state cluster1 mirror image_4 started 2024-09-14T08:54:19.634 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:54:19.634 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:19.634 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_4 2024-09-14T08:54:19.634 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=started 2024-09-14T08:54:19.634 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_result 2024-09-14T08:54:19.634 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=stopped 2024-09-14T08:54:19.634 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_4 2024-09-14T08:54:19.634 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_instance=cluster1 2024-09-14T08:54:19.634 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:54:19.635 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T08:54:19.635 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ grep -i state 2024-09-14T08:54:19.635 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance=cluster1 2024-09-14T08:54:19.635 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local loop_instance 2024-09-14T08:54:19.635 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:54:19.635 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sleep 0 2024-09-14T08:54:19.638 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T08:54:19.638 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ seq 0 3 2024-09-14T08:54:19.638 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:54:19.638 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_4 2024-09-14T08:54:19.638 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:0 2024-09-14T08:54:19.638 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:54:19.638 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:54:19.638 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T08:54:19.638 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:0 2024-09-14T08:54:19.638 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:54:19.639 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:54:19.639 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:54:19.639 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=0 2024-09-14T08:54:19.639 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T08:54:19.639 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:54:19.639 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:54:19.639 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=0 2024-09-14T08:54:19.639 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:54:19.639 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T08:54:19.639 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:0 2024-09-14T08:54:19.639 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:54:19.639 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:54:19.639 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T08:54:19.640 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:0 2024-09-14T08:54:19.640 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:54:19.640 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:54:19.640 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:54:19.640 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:54:19.640 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T08:54:19.640 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:54:19.640 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:54:19.640 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=0 2024-09-14T08:54:19.640 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=0 2024-09-14T08:54:19.640 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T08:54:19.655 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:54:19.656 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:54:19.656 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:54:19.656 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_4 2024-09-14T08:54:19.736 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:no valid command found; 3 closest matches: 2024-09-14T08:54:19.737 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status mirror/image_40 2024-09-14T08:54:19.737 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status mirror/image_44 2024-09-14T08:54:19.737 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status mirror/image_48 2024-09-14T08:54:19.737 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:54:19.740 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:54:19.740 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_4 2024-09-14T08:54:19.740 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:1 2024-09-14T08:54:19.740 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:54:19.740 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:54:19.740 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T08:54:19.740 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:1 2024-09-14T08:54:19.740 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:54:19.740 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:54:19.740 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:54:19.741 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=1 2024-09-14T08:54:19.741 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T08:54:19.741 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:54:19.741 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:54:19.741 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=1 2024-09-14T08:54:19.741 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:54:19.741 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T08:54:19.741 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:1 2024-09-14T08:54:19.741 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:54:19.741 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:54:19.741 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T08:54:19.741 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:1 2024-09-14T08:54:19.741 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:54:19.741 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:54:19.742 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:54:19.742 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:54:19.742 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T08:54:19.742 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:54:19.742 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:54:19.742 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=1 2024-09-14T08:54:19.742 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=1 2024-09-14T08:54:19.742 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T08:54:19.759 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:54:19.759 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:54:19.759 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:54:19.759 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_4 2024-09-14T08:54:19.840 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:no valid command found; 2 closest matches: 2024-09-14T08:54:19.840 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status mirror/image_43 2024-09-14T08:54:19.840 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status mirror/image_47 2024-09-14T08:54:19.840 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:54:19.843 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:54:19.843 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_4 2024-09-14T08:54:19.844 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:2 2024-09-14T08:54:19.844 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:54:19.844 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:54:19.844 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-14T08:54:19.844 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:2 2024-09-14T08:54:19.844 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:54:19.844 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:54:19.844 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:54:19.844 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=2 2024-09-14T08:54:19.844 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T08:54:19.844 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:54:19.844 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:54:19.844 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=2 2024-09-14T08:54:19.845 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:54:19.845 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-14T08:54:19.845 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:2 2024-09-14T08:54:19.845 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:54:19.845 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:54:19.845 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T08:54:19.845 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:2 2024-09-14T08:54:19.845 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:54:19.845 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:54:19.845 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:54:19.845 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:54:19.845 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-14T08:54:19.846 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:54:19.846 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:54:19.846 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=2 2024-09-14T08:54:19.846 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=2 2024-09-14T08:54:19.846 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T08:54:19.862 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:54:19.862 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:54:19.862 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:54:19.862 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_4 2024-09-14T08:54:19.950 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ return 0 2024-09-14T08:54:19.951 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T08:54:19.951 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ echo ' "state": "Replaying"' 2024-09-14T08:54:19.951 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep -i Replaying 2024-09-14T08:54:19.952 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: "state": "Replaying" 2024-09-14T08:54:19.954 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_state=started 2024-09-14T08:54:19.954 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test started = started 2024-09-14T08:54:19.954 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:54:19.954 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_4 2024-09-14T08:54:19.954 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:54:19.954 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:54:19.954 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:19.954 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_4 2024-09-14T08:54:19.954 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = journal ']' 2024-09-14T08:54:19.955 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = snapshot ']' 2024-09-14T08:54:19.955 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_4 2024-09-14T08:54:19.955 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:54:19.955 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:54:19.955 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:19.955 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_4 2024-09-14T08:54:19.955 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster2-mirror-image_4.status 2024-09-14T08:54:19.955 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster2-mirror-image_4.status 2024-09-14T08:54:19.955 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:54:19.956 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_4.status 2024-09-14T08:54:19.956 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster1-mirror-image_4.status 2024-09-14T08:54:19.957 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster1-mirror-image_4.status 2024-09-14T08:54:19.957 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:54:19.957 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_status_log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_4.status 2024-09-14T08:54:19.957 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ mirror_image_snapshot cluster2 mirror image_4 2024-09-14T08:54:19.957 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:54:19.957 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:19.957 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_4 2024-09-14T08:54:19.957 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_4 2024-09-14T08:54:20.653 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Snapshot ID: 92 2024-09-14T08:54:20.660 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_4 /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_4.status 2024-09-14T08:54:20.660 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:54:20.660 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:20.660 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_4 2024-09-14T08:54:20.660 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_4.status 2024-09-14T08:54:20.660 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_4 --format xml 2024-09-14T08:54:20.660 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:54:20.708 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-14T08:54:20.710 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snapshot_id=92 2024-09-14T08:54:20.711 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:54:20.711 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:54:20.711 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.2 2024-09-14T08:54:20.912 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_4 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_4.status 2024-09-14T08:54:20.912 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:54:20.912 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:20.912 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_4 2024-09-14T08:54:20.912 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_4.status 2024-09-14T08:54:20.912 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_4 --format xml 2024-09-14T08:54:20.912 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:54:20.963 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:54:20.965 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=38 2024-09-14T08:54:20.965 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 92 = 38 2024-09-14T08:54:20.965 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:54:20.965 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.4 2024-09-14T08:54:21.366 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_4 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_4.status 2024-09-14T08:54:21.366 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:54:21.366 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:21.367 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_4 2024-09-14T08:54:21.367 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_4.status 2024-09-14T08:54:21.367 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_4 --format xml 2024-09-14T08:54:21.367 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:54:21.417 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:54:21.418 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:54:21.419 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:54:21.419 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:54:21.419 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:54:21.419 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:54:21.419 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 92 = '' 2024-09-14T08:54:21.420 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:54:21.420 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.8 2024-09-14T08:54:22.221 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_4 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_4.status 2024-09-14T08:54:22.221 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:54:22.221 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:22.221 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_4 2024-09-14T08:54:22.221 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_4.status 2024-09-14T08:54:22.221 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_4 --format xml 2024-09-14T08:54:22.221 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:54:22.276 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:54:22.276 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:54:22.278 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:54:22.278 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:54:22.278 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:54:22.278 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:54:22.278 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 92 = '' 2024-09-14T08:54:22.279 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:54:22.279 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1.6 2024-09-14T08:54:23.880 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_4 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_4.status 2024-09-14T08:54:23.880 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:54:23.880 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:23.880 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_4 2024-09-14T08:54:23.880 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_4.status 2024-09-14T08:54:23.880 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_4 --format xml 2024-09-14T08:54:23.881 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:54:23.934 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:54:23.936 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=92 2024-09-14T08:54:23.937 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 92 = 92 2024-09-14T08:54:23.937 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:54:23.937 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snap_present cluster1 mirror image_4 snap 2024-09-14T08:54:23.937 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:54:23.937 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:23.937 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_4 2024-09-14T08:54:23.937 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:54:23.937 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:54:23.937 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T08:54:23.937 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:54:24.938 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 info mirror/image_4@snap 2024-09-14T08:54:24.985 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_4': 2024-09-14T08:54:24.985 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T08:54:24.986 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:54:24.986 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 2 2024-09-14T08:54:24.986 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 10bda0847b8e 2024-09-14T08:54:24.986 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.10bda0847b8e 2024-09-14T08:54:24.986 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:54:24.986 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering, non-primary 2024-09-14T08:54:24.986 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:54:24.986 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:54:24.986 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:45:52 2024 2024-09-14T08:54:24.986 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:45:52 2024 2024-09-14T08:54:24.986 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:45:52 2024 2024-09-14T08:54:24.986 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: protected: False 2024-09-14T08:54:24.986 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:54:24.986 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:54:24.987 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: 3fcd7b58-faf2-4823-8a72-3bd4c17d6db0 2024-09-14T08:54:24.987 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: false 2024-09-14T08:54:24.989 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:54:24.990 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ compare_image_snaps mirror image_4 snap 2024-09-14T08:54:24.990 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:24.990 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_4 2024-09-14T08:54:24.990 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:54:24.990 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local ret=0 2024-09-14T08:54:24.990 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local rmt_export=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_4.export 2024-09-14T08:54:24.990 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local loc_export=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_4.export 2024-09-14T08:54:24.990 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_4.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_4.export 2024-09-14T08:54:24.990 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror export image_4@snap /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_4.export 2024-09-14T08:54:25.188 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T08:54:25.195 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 -p mirror export image_4@snap /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_4.export 2024-09-14T08:54:25.444 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T08:54:25.451 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cmp /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_4.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_4.export 2024-09-14T08:54:25.505 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_4.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_4.export 2024-09-14T08:54:25.537 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:54:25.538 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T08:54:25.538 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_5 2024-09-14T08:54:25.538 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_snap cluster2 mirror image_5 snap 2024-09-14T08:54:25.538 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:54:25.538 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:25.538 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_5 2024-09-14T08:54:25.538 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:54:25.538 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror snap create image_5@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T08:54:26.190 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_started cluster1 mirror image_5 2024-09-14T08:54:26.190 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:54:26.190 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:26.190 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_5 2024-09-14T08:54:26.190 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_state cluster1 mirror image_5 started 2024-09-14T08:54:26.190 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:54:26.190 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:26.190 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_5 2024-09-14T08:54:26.191 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=started 2024-09-14T08:54:26.191 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:54:26.191 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:54:26.191 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:54:27.190 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_replay_state cluster1 mirror image_5 started 2024-09-14T08:54:27.191 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:54:27.191 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:27.191 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_5 2024-09-14T08:54:27.191 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=started 2024-09-14T08:54:27.191 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_result 2024-09-14T08:54:27.191 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=stopped 2024-09-14T08:54:27.192 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_5 2024-09-14T08:54:27.192 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_instance=cluster1 2024-09-14T08:54:27.192 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:54:27.193 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ grep -i state 2024-09-14T08:54:27.193 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T08:54:27.193 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance=cluster1 2024-09-14T08:54:27.193 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local loop_instance 2024-09-14T08:54:27.193 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:54:27.193 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sleep 0 2024-09-14T08:54:27.193 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T08:54:27.193 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ seq 0 3 2024-09-14T08:54:27.195 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:54:27.195 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_5 2024-09-14T08:54:27.195 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:0 2024-09-14T08:54:27.195 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:54:27.195 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:54:27.195 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T08:54:27.195 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:0 2024-09-14T08:54:27.195 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:54:27.195 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:54:27.195 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:54:27.195 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=0 2024-09-14T08:54:27.195 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T08:54:27.196 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:54:27.196 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:54:27.196 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=0 2024-09-14T08:54:27.196 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:54:27.196 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T08:54:27.196 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:0 2024-09-14T08:54:27.196 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:54:27.196 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:54:27.197 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T08:54:27.198 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:0 2024-09-14T08:54:27.198 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:54:27.198 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:54:27.198 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:54:27.198 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:54:27.198 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T08:54:27.198 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:54:27.198 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:54:27.198 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=0 2024-09-14T08:54:27.198 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=0 2024-09-14T08:54:27.198 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T08:54:27.216 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:54:27.216 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:54:27.216 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:54:27.216 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_5 2024-09-14T08:54:27.310 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ return 0 2024-09-14T08:54:27.311 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T08:54:27.311 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ echo ' "state": "Replaying"' 2024-09-14T08:54:27.311 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep -i Replaying 2024-09-14T08:54:27.312 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: "state": "Replaying" 2024-09-14T08:54:27.317 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_state=started 2024-09-14T08:54:27.317 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test started = started 2024-09-14T08:54:27.317 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:54:27.317 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_5 2024-09-14T08:54:27.318 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:54:27.318 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:54:27.318 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:27.318 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_5 2024-09-14T08:54:27.318 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = journal ']' 2024-09-14T08:54:27.318 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = snapshot ']' 2024-09-14T08:54:27.318 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_5 2024-09-14T08:54:27.318 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:54:27.318 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:54:27.318 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:27.318 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_5 2024-09-14T08:54:27.318 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster2-mirror-image_5.status 2024-09-14T08:54:27.318 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster2-mirror-image_5.status 2024-09-14T08:54:27.319 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:54:27.319 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_5.status 2024-09-14T08:54:27.319 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster1-mirror-image_5.status 2024-09-14T08:54:27.319 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster1-mirror-image_5.status 2024-09-14T08:54:27.319 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:54:27.319 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_status_log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_5.status 2024-09-14T08:54:27.319 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ mirror_image_snapshot cluster2 mirror image_5 2024-09-14T08:54:27.319 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:54:27.319 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:27.319 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_5 2024-09-14T08:54:27.319 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_5 2024-09-14T08:54:28.203 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Snapshot ID: 94 2024-09-14T08:54:28.210 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_5 /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_5.status 2024-09-14T08:54:28.210 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:54:28.210 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:28.210 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_5 2024-09-14T08:54:28.210 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_5.status 2024-09-14T08:54:28.211 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_5 --format xml 2024-09-14T08:54:28.211 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:54:28.264 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-14T08:54:28.266 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snapshot_id=94 2024-09-14T08:54:28.266 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:54:28.266 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:54:28.266 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.2 2024-09-14T08:54:28.468 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_5 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_5.status 2024-09-14T08:54:28.468 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:54:28.468 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:28.468 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_5 2024-09-14T08:54:28.468 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_5.status 2024-09-14T08:54:28.468 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_5 --format xml 2024-09-14T08:54:28.468 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:54:28.516 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:54:28.517 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:54:28.518 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:54:28.518 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:54:28.518 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:54:28.519 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:54:28.519 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 94 = '' 2024-09-14T08:54:28.519 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:54:28.519 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.4 2024-09-14T08:54:28.920 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_5 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_5.status 2024-09-14T08:54:28.920 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:54:28.920 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:28.920 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_5 2024-09-14T08:54:28.921 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_5.status 2024-09-14T08:54:28.921 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_5 --format xml 2024-09-14T08:54:28.921 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:54:28.970 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:54:28.971 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:54:28.972 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:54:28.972 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:54:28.972 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:54:28.973 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:54:28.973 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 94 = '' 2024-09-14T08:54:28.973 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:54:28.973 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.8 2024-09-14T08:54:29.774 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_5 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_5.status 2024-09-14T08:54:29.774 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:54:29.774 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:29.774 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_5 2024-09-14T08:54:29.774 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_5.status 2024-09-14T08:54:29.775 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_5 --format xml 2024-09-14T08:54:29.775 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:54:29.821 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:54:29.823 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=94 2024-09-14T08:54:29.823 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 94 = 94 2024-09-14T08:54:29.824 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:54:29.824 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snap_present cluster1 mirror image_5 snap 2024-09-14T08:54:29.824 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:54:29.824 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:29.824 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_5 2024-09-14T08:54:29.824 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:54:29.824 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:54:29.824 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T08:54:29.824 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:54:30.824 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 info mirror/image_5@snap 2024-09-14T08:54:30.871 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_5': 2024-09-14T08:54:30.871 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T08:54:30.871 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:54:30.871 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 2 2024-09-14T08:54:30.871 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 127e41b63218 2024-09-14T08:54:30.871 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.127e41b63218 2024-09-14T08:54:30.871 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:54:30.871 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering, non-primary 2024-09-14T08:54:30.871 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:54:30.871 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:54:30.872 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:45:51 2024 2024-09-14T08:54:30.872 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:45:51 2024 2024-09-14T08:54:30.872 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:45:51 2024 2024-09-14T08:54:30.872 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: protected: False 2024-09-14T08:54:30.872 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:54:30.872 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:54:30.872 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: 98c80276-e6b6-4a70-b145-47505e1b7690 2024-09-14T08:54:30.872 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: false 2024-09-14T08:54:30.875 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:54:30.875 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ compare_image_snaps mirror image_5 snap 2024-09-14T08:54:30.875 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:30.875 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_5 2024-09-14T08:54:30.875 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:54:30.875 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local ret=0 2024-09-14T08:54:30.875 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local rmt_export=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_5.export 2024-09-14T08:54:30.875 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local loc_export=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_5.export 2024-09-14T08:54:30.875 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_5.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_5.export 2024-09-14T08:54:30.876 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror export image_5@snap /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_5.export 2024-09-14T08:54:31.079 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T08:54:31.086 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 -p mirror export image_5@snap /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_5.export 2024-09-14T08:54:31.331 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T08:54:31.339 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cmp /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_5.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_5.export 2024-09-14T08:54:31.394 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_5.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_5.export 2024-09-14T08:54:31.424 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:54:31.425 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T08:54:31.425 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_6 2024-09-14T08:54:31.425 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_snap cluster2 mirror image_6 snap 2024-09-14T08:54:31.425 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:54:31.425 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:31.425 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_6 2024-09-14T08:54:31.425 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:54:31.425 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror snap create image_6@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T08:54:32.203 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_started cluster1 mirror image_6 2024-09-14T08:54:32.203 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:54:32.203 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:32.203 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_6 2024-09-14T08:54:32.203 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_state cluster1 mirror image_6 started 2024-09-14T08:54:32.203 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:54:32.204 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:32.204 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_6 2024-09-14T08:54:32.204 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=started 2024-09-14T08:54:32.204 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:54:32.204 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:54:32.204 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:54:33.204 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_replay_state cluster1 mirror image_6 started 2024-09-14T08:54:33.204 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:54:33.205 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:33.205 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_6 2024-09-14T08:54:33.205 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=started 2024-09-14T08:54:33.205 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_result 2024-09-14T08:54:33.205 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=stopped 2024-09-14T08:54:33.206 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_6 2024-09-14T08:54:33.206 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_instance=cluster1 2024-09-14T08:54:33.206 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ grep -i state 2024-09-14T08:54:33.206 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:54:33.206 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T08:54:33.206 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance=cluster1 2024-09-14T08:54:33.206 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local loop_instance 2024-09-14T08:54:33.207 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:54:33.207 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sleep 0 2024-09-14T08:54:33.207 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T08:54:33.207 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ seq 0 3 2024-09-14T08:54:33.209 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:54:33.210 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_6 2024-09-14T08:54:33.210 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:0 2024-09-14T08:54:33.210 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:54:33.210 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:54:33.210 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T08:54:33.210 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:0 2024-09-14T08:54:33.210 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:54:33.210 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:54:33.210 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:54:33.210 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=0 2024-09-14T08:54:33.210 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T08:54:33.210 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:54:33.210 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:54:33.210 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=0 2024-09-14T08:54:33.211 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:54:33.211 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T08:54:33.211 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:0 2024-09-14T08:54:33.211 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:54:33.211 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:54:33.211 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T08:54:33.211 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:0 2024-09-14T08:54:33.211 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:54:33.211 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:54:33.211 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:54:33.211 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:54:33.211 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T08:54:33.211 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:54:33.211 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:54:33.212 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=0 2024-09-14T08:54:33.212 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=0 2024-09-14T08:54:33.212 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T08:54:33.231 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:54:33.231 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:54:33.231 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:54:33.231 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_6 2024-09-14T08:54:33.320 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_6'] 2024-09-14T08:54:33.320 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:54:33.320 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:54:33.323 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:54:33.323 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_6 2024-09-14T08:54:33.323 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:1 2024-09-14T08:54:33.323 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:54:33.323 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:54:33.323 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T08:54:33.324 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:1 2024-09-14T08:54:33.324 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:54:33.324 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:54:33.324 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:54:33.324 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=1 2024-09-14T08:54:33.324 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T08:54:33.324 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:54:33.324 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:54:33.324 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=1 2024-09-14T08:54:33.324 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:54:33.324 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T08:54:33.324 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:1 2024-09-14T08:54:33.324 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:54:33.324 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:54:33.325 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T08:54:33.325 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:1 2024-09-14T08:54:33.325 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:54:33.325 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:54:33.325 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:54:33.325 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:54:33.325 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T08:54:33.325 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:54:33.325 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:54:33.325 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=1 2024-09-14T08:54:33.325 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=1 2024-09-14T08:54:33.325 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T08:54:33.342 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:54:33.342 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:54:33.342 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:54:33.342 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_6 2024-09-14T08:54:33.425 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_6'] 2024-09-14T08:54:33.425 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:54:33.425 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:54:33.429 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:54:33.429 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_6 2024-09-14T08:54:33.429 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:2 2024-09-14T08:54:33.429 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:54:33.429 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:54:33.429 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-14T08:54:33.429 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:2 2024-09-14T08:54:33.429 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:54:33.429 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:54:33.429 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:54:33.429 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=2 2024-09-14T08:54:33.429 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T08:54:33.430 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:54:33.430 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:54:33.430 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=2 2024-09-14T08:54:33.430 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:54:33.430 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-14T08:54:33.430 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:2 2024-09-14T08:54:33.430 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:54:33.430 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:54:33.430 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T08:54:33.430 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:2 2024-09-14T08:54:33.430 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:54:33.430 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:54:33.430 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:54:33.430 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:54:33.431 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-14T08:54:33.431 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:54:33.431 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:54:33.431 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=2 2024-09-14T08:54:33.431 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=2 2024-09-14T08:54:33.431 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T08:54:33.449 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:54:33.449 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:54:33.449 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:54:33.449 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_6 2024-09-14T08:54:33.532 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_6'] 2024-09-14T08:54:33.532 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:54:33.532 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:54:33.535 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:54:33.535 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_6 2024-09-14T08:54:33.535 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:3 2024-09-14T08:54:33.535 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:54:33.536 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:54:33.536 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:3 cluster instance 2024-09-14T08:54:33.536 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:3 2024-09-14T08:54:33.536 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:54:33.536 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:54:33.536 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:54:33.536 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=3 2024-09-14T08:54:33.536 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-14T08:54:33.536 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:54:33.536 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:54:33.536 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=3 2024-09-14T08:54:33.536 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=3 2024-09-14T08:54:33.537 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:3 cluster1 2024-09-14T08:54:33.537 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:3 2024-09-14T08:54:33.537 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:54:33.537 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:54:33.537 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-14T08:54:33.537 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:3 2024-09-14T08:54:33.537 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:54:33.537 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:54:33.537 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:54:33.537 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=3 2024-09-14T08:54:33.537 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 3 = cluster1:3 ']' 2024-09-14T08:54:33.538 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:54:33.538 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:54:33.538 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=3 2024-09-14T08:54:33.538 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=3 2024-09-14T08:54:33.538 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-14T08:54:33.553 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T08:54:33.553 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T08:54:33.554 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T08:54:33.554 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_6 2024-09-14T08:54:33.645 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ return 0 2024-09-14T08:54:33.646 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T08:54:33.646 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ echo ' "state": "Replaying"' 2024-09-14T08:54:33.646 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep -i Replaying 2024-09-14T08:54:33.647 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: "state": "Replaying" 2024-09-14T08:54:33.649 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_state=started 2024-09-14T08:54:33.649 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test started = started 2024-09-14T08:54:33.649 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:54:33.649 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_6 2024-09-14T08:54:33.649 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:54:33.649 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:54:33.649 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:33.649 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_6 2024-09-14T08:54:33.649 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = journal ']' 2024-09-14T08:54:33.650 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = snapshot ']' 2024-09-14T08:54:33.650 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_6 2024-09-14T08:54:33.650 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:54:33.650 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:54:33.650 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:33.650 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_6 2024-09-14T08:54:33.650 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster2-mirror-image_6.status 2024-09-14T08:54:33.650 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster2-mirror-image_6.status 2024-09-14T08:54:33.650 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:54:33.650 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_6.status 2024-09-14T08:54:33.651 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster1-mirror-image_6.status 2024-09-14T08:54:33.651 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster1-mirror-image_6.status 2024-09-14T08:54:33.652 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:54:33.652 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_status_log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_6.status 2024-09-14T08:54:33.652 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ mirror_image_snapshot cluster2 mirror image_6 2024-09-14T08:54:33.652 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:54:33.652 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:33.652 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_6 2024-09-14T08:54:33.652 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_6 2024-09-14T08:54:34.218 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Snapshot ID: 96 2024-09-14T08:54:34.231 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_6 /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_6.status 2024-09-14T08:54:34.231 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:54:34.231 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:34.231 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_6 2024-09-14T08:54:34.231 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_6.status 2024-09-14T08:54:34.231 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_6 --format xml 2024-09-14T08:54:34.231 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:54:34.277 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-14T08:54:34.279 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snapshot_id=96 2024-09-14T08:54:34.279 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:54:34.279 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:54:34.279 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.2 2024-09-14T08:54:34.480 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_6 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_6.status 2024-09-14T08:54:34.481 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:54:34.481 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:34.481 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_6 2024-09-14T08:54:34.481 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_6.status 2024-09-14T08:54:34.481 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_6 --format xml 2024-09-14T08:54:34.481 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:54:34.529 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:54:34.529 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:54:34.531 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:54:34.531 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:54:34.531 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:54:34.531 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:54:34.531 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 96 = '' 2024-09-14T08:54:34.531 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:54:34.531 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.4 2024-09-14T08:54:34.932 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_6 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_6.status 2024-09-14T08:54:34.932 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:54:34.932 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:34.933 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_6 2024-09-14T08:54:34.933 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_6.status 2024-09-14T08:54:34.933 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_6 --format xml 2024-09-14T08:54:34.933 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:54:34.984 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:54:34.985 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:54:34.986 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:54:34.986 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:54:34.986 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:54:34.986 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:54:34.987 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 96 = '' 2024-09-14T08:54:34.987 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:54:34.987 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.8 2024-09-14T08:54:35.788 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_6 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_6.status 2024-09-14T08:54:35.788 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:54:35.788 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:35.788 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_6 2024-09-14T08:54:35.788 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_6.status 2024-09-14T08:54:35.789 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_6 --format xml 2024-09-14T08:54:35.789 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:54:35.835 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:54:35.837 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=96 2024-09-14T08:54:35.837 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 96 = 96 2024-09-14T08:54:35.838 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:54:35.838 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snap_present cluster1 mirror image_6 snap 2024-09-14T08:54:35.838 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:54:35.838 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:35.838 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_6 2024-09-14T08:54:35.838 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:54:35.838 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:54:35.838 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T08:54:35.838 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:54:36.838 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 info mirror/image_6@snap 2024-09-14T08:54:36.885 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_6': 2024-09-14T08:54:36.885 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T08:54:36.885 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:54:36.885 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 2 2024-09-14T08:54:36.885 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 10c2c489d747 2024-09-14T08:54:36.885 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.10c2c489d747 2024-09-14T08:54:36.885 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:54:36.885 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering, non-primary 2024-09-14T08:54:36.885 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:54:36.886 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:54:36.886 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:46:04 2024 2024-09-14T08:54:36.886 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:46:04 2024 2024-09-14T08:54:36.886 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:46:04 2024 2024-09-14T08:54:36.886 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: protected: False 2024-09-14T08:54:36.886 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:54:36.886 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:54:36.886 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: 82e82fc9-f6e1-4a89-9145-f0e06f4942c1 2024-09-14T08:54:36.886 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: false 2024-09-14T08:54:36.889 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:54:36.889 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ compare_image_snaps mirror image_6 snap 2024-09-14T08:54:36.889 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:36.890 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_6 2024-09-14T08:54:36.890 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:54:36.890 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local ret=0 2024-09-14T08:54:36.890 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local rmt_export=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_6.export 2024-09-14T08:54:36.890 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local loc_export=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_6.export 2024-09-14T08:54:36.890 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_6.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_6.export 2024-09-14T08:54:36.890 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror export image_6@snap /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_6.export 2024-09-14T08:54:37.083 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T08:54:37.090 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 -p mirror export image_6@snap /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_6.export 2024-09-14T08:54:37.346 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T08:54:37.353 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cmp /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_6.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_6.export 2024-09-14T08:54:37.409 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_6.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_6.export 2024-09-14T08:54:37.440 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:54:37.440 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T08:54:37.441 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_7 2024-09-14T08:54:37.441 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_snap cluster2 mirror image_7 snap 2024-09-14T08:54:37.441 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:54:37.441 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:37.441 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_7 2024-09-14T08:54:37.441 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:54:37.441 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror snap create image_7@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T08:54:38.224 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_started cluster1 mirror image_7 2024-09-14T08:54:38.224 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:54:38.224 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:38.225 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_7 2024-09-14T08:54:38.225 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_state cluster1 mirror image_7 started 2024-09-14T08:54:38.225 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:54:38.225 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:38.225 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_7 2024-09-14T08:54:38.225 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=started 2024-09-14T08:54:38.225 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:54:38.225 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:54:38.225 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:54:39.225 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_replay_state cluster1 mirror image_7 started 2024-09-14T08:54:39.225 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:54:39.226 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:39.226 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_7 2024-09-14T08:54:39.226 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=started 2024-09-14T08:54:39.226 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_result 2024-09-14T08:54:39.226 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=stopped 2024-09-14T08:54:39.227 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_7 2024-09-14T08:54:39.227 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_instance=cluster1 2024-09-14T08:54:39.227 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:54:39.227 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ grep -i state 2024-09-14T08:54:39.227 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T08:54:39.227 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance=cluster1 2024-09-14T08:54:39.227 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local loop_instance 2024-09-14T08:54:39.227 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:54:39.227 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sleep 0 2024-09-14T08:54:39.227 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T08:54:39.228 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ seq 0 3 2024-09-14T08:54:39.230 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:54:39.230 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_7 2024-09-14T08:54:39.230 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:0 2024-09-14T08:54:39.230 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:54:39.230 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:54:39.230 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T08:54:39.230 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:0 2024-09-14T08:54:39.230 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:54:39.230 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:54:39.231 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:54:39.231 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=0 2024-09-14T08:54:39.231 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T08:54:39.231 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:54:39.231 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:54:39.231 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=0 2024-09-14T08:54:39.231 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:54:39.231 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T08:54:39.231 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:0 2024-09-14T08:54:39.231 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:54:39.231 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:54:39.232 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T08:54:39.232 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:0 2024-09-14T08:54:39.232 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:54:39.232 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:54:39.232 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:54:39.232 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:54:39.232 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T08:54:39.233 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:54:39.233 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:54:39.233 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=0 2024-09-14T08:54:39.233 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=0 2024-09-14T08:54:39.233 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T08:54:39.253 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:54:39.253 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:54:39.253 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:54:39.253 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_7 2024-09-14T08:54:39.340 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_7'] 2024-09-14T08:54:39.340 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:54:39.340 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:54:39.343 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:54:39.343 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_7 2024-09-14T08:54:39.343 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:1 2024-09-14T08:54:39.343 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:54:39.343 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:54:39.343 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T08:54:39.343 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:1 2024-09-14T08:54:39.343 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:54:39.343 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:54:39.343 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:54:39.343 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=1 2024-09-14T08:54:39.344 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T08:54:39.344 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:54:39.344 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:54:39.344 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=1 2024-09-14T08:54:39.344 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:54:39.344 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T08:54:39.344 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:1 2024-09-14T08:54:39.344 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:54:39.344 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:54:39.344 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T08:54:39.344 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:1 2024-09-14T08:54:39.344 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:54:39.344 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:54:39.344 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:54:39.344 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:54:39.345 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T08:54:39.345 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:54:39.345 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:54:39.345 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=1 2024-09-14T08:54:39.345 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=1 2024-09-14T08:54:39.345 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T08:54:39.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:54:39.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:54:39.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:54:39.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_7 2024-09-14T08:54:39.444 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_7'] 2024-09-14T08:54:39.445 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:54:39.445 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:54:39.447 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:54:39.447 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_7 2024-09-14T08:54:39.448 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:2 2024-09-14T08:54:39.448 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:54:39.448 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:54:39.448 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-14T08:54:39.448 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:2 2024-09-14T08:54:39.448 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:54:39.448 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:54:39.448 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:54:39.448 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=2 2024-09-14T08:54:39.448 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T08:54:39.448 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:54:39.448 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:54:39.449 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=2 2024-09-14T08:54:39.449 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:54:39.449 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-14T08:54:39.449 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:2 2024-09-14T08:54:39.449 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:54:39.449 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:54:39.449 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T08:54:39.449 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:2 2024-09-14T08:54:39.449 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:54:39.449 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:54:39.449 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:54:39.449 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:54:39.449 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-14T08:54:39.450 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:54:39.450 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:54:39.450 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=2 2024-09-14T08:54:39.450 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=2 2024-09-14T08:54:39.450 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T08:54:39.467 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:54:39.467 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:54:39.467 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:54:39.467 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_7 2024-09-14T08:54:39.549 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_7'] 2024-09-14T08:54:39.549 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:54:39.549 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:54:39.552 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:54:39.552 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_7 2024-09-14T08:54:39.552 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:3 2024-09-14T08:54:39.552 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:54:39.552 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:54:39.552 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:3 cluster instance 2024-09-14T08:54:39.552 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:3 2024-09-14T08:54:39.552 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:54:39.552 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:54:39.552 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:54:39.552 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=3 2024-09-14T08:54:39.553 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-14T08:54:39.553 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:54:39.553 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:54:39.553 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=3 2024-09-14T08:54:39.553 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=3 2024-09-14T08:54:39.553 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:3 cluster1 2024-09-14T08:54:39.553 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:3 2024-09-14T08:54:39.553 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:54:39.553 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:54:39.553 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-14T08:54:39.553 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:3 2024-09-14T08:54:39.553 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:54:39.553 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:54:39.553 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:54:39.554 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=3 2024-09-14T08:54:39.554 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 3 = cluster1:3 ']' 2024-09-14T08:54:39.554 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:54:39.554 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:54:39.554 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=3 2024-09-14T08:54:39.554 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=3 2024-09-14T08:54:39.554 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-14T08:54:39.570 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T08:54:39.570 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T08:54:39.571 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T08:54:39.571 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_7 2024-09-14T08:54:39.660 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ return 0 2024-09-14T08:54:39.661 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T08:54:39.661 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ echo ' "state": "Replaying"' 2024-09-14T08:54:39.661 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep -i Replaying 2024-09-14T08:54:39.662 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: "state": "Replaying" 2024-09-14T08:54:39.663 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_state=started 2024-09-14T08:54:39.663 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test started = started 2024-09-14T08:54:39.663 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:54:39.663 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_7 2024-09-14T08:54:39.663 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:54:39.664 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:54:39.664 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:39.664 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_7 2024-09-14T08:54:39.664 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = journal ']' 2024-09-14T08:54:39.664 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = snapshot ']' 2024-09-14T08:54:39.664 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_7 2024-09-14T08:54:39.664 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:54:39.664 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:54:39.664 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:39.664 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_7 2024-09-14T08:54:39.665 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster2-mirror-image_7.status 2024-09-14T08:54:39.665 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster2-mirror-image_7.status 2024-09-14T08:54:39.665 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:54:39.666 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_7.status 2024-09-14T08:54:39.666 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster1-mirror-image_7.status 2024-09-14T08:54:39.666 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster1-mirror-image_7.status 2024-09-14T08:54:39.666 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:54:39.666 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_status_log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_7.status 2024-09-14T08:54:39.666 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ mirror_image_snapshot cluster2 mirror image_7 2024-09-14T08:54:39.666 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:54:39.666 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:39.666 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_7 2024-09-14T08:54:39.667 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_7 2024-09-14T08:54:40.232 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Snapshot ID: 98 2024-09-14T08:54:40.238 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_7 /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_7.status 2024-09-14T08:54:40.238 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:54:40.238 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:40.238 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_7 2024-09-14T08:54:40.239 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_7.status 2024-09-14T08:54:40.239 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_7 --format xml 2024-09-14T08:54:40.239 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:54:40.285 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-14T08:54:40.286 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snapshot_id=98 2024-09-14T08:54:40.286 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:54:40.286 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:54:40.286 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.2 2024-09-14T08:54:40.488 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_7 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_7.status 2024-09-14T08:54:40.488 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:54:40.488 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:40.488 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_7 2024-09-14T08:54:40.488 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_7.status 2024-09-14T08:54:40.488 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_7 --format xml 2024-09-14T08:54:40.488 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:54:40.537 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:54:40.538 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:54:40.538 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:54:40.539 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:54:40.539 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:54:40.539 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:54:40.539 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 98 = '' 2024-09-14T08:54:40.541 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:54:40.541 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.4 2024-09-14T08:54:40.941 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_7 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_7.status 2024-09-14T08:54:40.941 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:54:40.941 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:40.941 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_7 2024-09-14T08:54:40.941 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_7.status 2024-09-14T08:54:40.941 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_7 --format xml 2024-09-14T08:54:40.941 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:54:40.988 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:54:40.988 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:54:40.989 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:54:40.989 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:54:40.990 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:54:40.990 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:54:40.990 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 98 = '' 2024-09-14T08:54:40.990 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:54:40.990 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.8 2024-09-14T08:54:41.791 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_7 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_7.status 2024-09-14T08:54:41.791 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:54:41.791 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:41.791 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_7 2024-09-14T08:54:41.791 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_7.status 2024-09-14T08:54:41.791 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_7 --format xml 2024-09-14T08:54:41.791 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:54:41.840 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:54:41.842 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=98 2024-09-14T08:54:41.843 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 98 = 98 2024-09-14T08:54:41.843 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:54:41.843 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snap_present cluster1 mirror image_7 snap 2024-09-14T08:54:41.843 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:54:41.843 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:41.843 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_7 2024-09-14T08:54:41.843 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:54:41.843 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:54:41.843 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T08:54:41.843 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:54:42.843 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 info mirror/image_7@snap 2024-09-14T08:54:42.888 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_7': 2024-09-14T08:54:42.888 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T08:54:42.888 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:54:42.888 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 2 2024-09-14T08:54:42.888 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 11893ab54979 2024-09-14T08:54:42.888 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.11893ab54979 2024-09-14T08:54:42.888 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:54:42.889 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering, non-primary 2024-09-14T08:54:42.889 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:54:42.889 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:54:42.889 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:46:10 2024 2024-09-14T08:54:42.889 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:46:10 2024 2024-09-14T08:54:42.889 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:46:10 2024 2024-09-14T08:54:42.889 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: protected: False 2024-09-14T08:54:42.889 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:54:42.889 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:54:42.889 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: 46c72cd0-0717-48e2-bd75-e9b03e063bb7 2024-09-14T08:54:42.889 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: false 2024-09-14T08:54:42.890 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:54:42.890 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ compare_image_snaps mirror image_7 snap 2024-09-14T08:54:42.891 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:42.891 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_7 2024-09-14T08:54:42.891 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:54:42.891 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local ret=0 2024-09-14T08:54:42.891 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local rmt_export=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_7.export 2024-09-14T08:54:42.891 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local loc_export=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_7.export 2024-09-14T08:54:42.891 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_7.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_7.export 2024-09-14T08:54:42.891 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror export image_7@snap /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_7.export 2024-09-14T08:54:43.077 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T08:54:43.084 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 -p mirror export image_7@snap /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_7.export 2024-09-14T08:54:43.348 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T08:54:43.355 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cmp /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_7.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_7.export 2024-09-14T08:54:43.411 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_7.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_7.export 2024-09-14T08:54:43.443 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:54:43.443 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T08:54:43.443 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_8 2024-09-14T08:54:43.443 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_snap cluster2 mirror image_8 snap 2024-09-14T08:54:43.443 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:54:43.443 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:43.444 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_8 2024-09-14T08:54:43.444 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:54:43.444 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror snap create image_8@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T08:54:44.246 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_started cluster1 mirror image_8 2024-09-14T08:54:44.246 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:54:44.246 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:44.246 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_8 2024-09-14T08:54:44.246 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_state cluster1 mirror image_8 started 2024-09-14T08:54:44.247 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:54:44.247 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:44.247 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_8 2024-09-14T08:54:44.247 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=started 2024-09-14T08:54:44.247 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:54:44.247 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:54:44.247 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:54:45.247 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_replay_state cluster1 mirror image_8 started 2024-09-14T08:54:45.247 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:54:45.248 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:45.248 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_8 2024-09-14T08:54:45.248 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=started 2024-09-14T08:54:45.248 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_result 2024-09-14T08:54:45.248 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=stopped 2024-09-14T08:54:45.249 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_8 2024-09-14T08:54:45.249 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_instance=cluster1 2024-09-14T08:54:45.249 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ grep -i state 2024-09-14T08:54:45.249 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:54:45.249 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T08:54:45.249 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance=cluster1 2024-09-14T08:54:45.250 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local loop_instance 2024-09-14T08:54:45.250 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:54:45.250 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sleep 0 2024-09-14T08:54:45.250 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T08:54:45.250 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ seq 0 3 2024-09-14T08:54:45.251 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:54:45.251 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_8 2024-09-14T08:54:45.251 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:0 2024-09-14T08:54:45.252 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:54:45.252 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:54:45.252 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T08:54:45.252 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:0 2024-09-14T08:54:45.252 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:54:45.252 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:54:45.252 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:54:45.252 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=0 2024-09-14T08:54:45.252 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T08:54:45.252 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:54:45.252 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:54:45.252 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=0 2024-09-14T08:54:45.252 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:54:45.254 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T08:54:45.254 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:0 2024-09-14T08:54:45.254 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:54:45.254 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:54:45.254 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T08:54:45.255 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:0 2024-09-14T08:54:45.255 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:54:45.255 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:54:45.255 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:54:45.255 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:54:45.255 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T08:54:45.255 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:54:45.255 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:54:45.255 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=0 2024-09-14T08:54:45.255 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=0 2024-09-14T08:54:45.255 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T08:54:45.275 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:54:45.275 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:54:45.276 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:54:45.276 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_8 2024-09-14T08:54:45.371 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ return 0 2024-09-14T08:54:45.372 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T08:54:45.372 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ echo ' "state": "Replaying"' 2024-09-14T08:54:45.372 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep -i Replaying 2024-09-14T08:54:45.373 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: "state": "Replaying" 2024-09-14T08:54:45.374 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_state=started 2024-09-14T08:54:45.374 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test started = started 2024-09-14T08:54:45.375 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:54:45.375 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_8 2024-09-14T08:54:45.375 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:54:45.375 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:54:45.375 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:45.375 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_8 2024-09-14T08:54:45.375 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = journal ']' 2024-09-14T08:54:45.375 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = snapshot ']' 2024-09-14T08:54:45.375 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_8 2024-09-14T08:54:45.375 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:54:45.375 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:54:45.375 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:45.375 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_8 2024-09-14T08:54:45.375 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster2-mirror-image_8.status 2024-09-14T08:54:45.375 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster2-mirror-image_8.status 2024-09-14T08:54:45.376 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:54:45.376 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_8.status 2024-09-14T08:54:45.376 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster1-mirror-image_8.status 2024-09-14T08:54:45.376 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster1-mirror-image_8.status 2024-09-14T08:54:45.376 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:54:45.377 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_status_log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_8.status 2024-09-14T08:54:45.377 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ mirror_image_snapshot cluster2 mirror image_8 2024-09-14T08:54:45.377 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:54:45.377 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:45.378 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_8 2024-09-14T08:54:45.378 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_8 2024-09-14T08:54:46.149 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.0 2024-09-14T08:54:46.150 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Sent signal 15 2024-09-14T08:54:46.150 INFO:tasks.rbd_mirror_thrash:waiting for 8 secs before reviving daemons 2024-09-14T08:54:46.151 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi164.stderr:2024-09-14T08:54:46.151+0000 7f5e75d1d640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 (PID: 48085) UID: 1000 2024-09-14T08:54:46.259 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Snapshot ID: 100 2024-09-14T08:54:46.265 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_8 /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_8.status 2024-09-14T08:54:46.265 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:54:46.265 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:46.265 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_8 2024-09-14T08:54:46.266 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_8.status 2024-09-14T08:54:46.266 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_8 --format xml 2024-09-14T08:54:46.266 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:54:46.311 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-14T08:54:46.313 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snapshot_id=100 2024-09-14T08:54:46.313 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:54:46.313 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:54:46.313 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.2 2024-09-14T08:54:46.515 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_8 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_8.status 2024-09-14T08:54:46.515 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:54:46.515 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:46.515 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_8 2024-09-14T08:54:46.515 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_8.status 2024-09-14T08:54:46.515 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_8 --format xml 2024-09-14T08:54:46.515 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:54:46.561 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:54:46.563 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=42 2024-09-14T08:54:46.563 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 100 = 42 2024-09-14T08:54:46.563 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:54:46.563 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.4 2024-09-14T08:54:46.965 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_8 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_8.status 2024-09-14T08:54:46.965 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:54:46.965 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:46.965 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_8 2024-09-14T08:54:46.965 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_8.status 2024-09-14T08:54:46.965 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_8 --format xml 2024-09-14T08:54:46.965 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:54:47.013 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:54:47.015 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=42 2024-09-14T08:54:47.015 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 100 = 42 2024-09-14T08:54:47.015 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:54:47.015 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.8 2024-09-14T08:54:47.816 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_8 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_8.status 2024-09-14T08:54:47.816 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:54:47.816 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:47.817 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_8 2024-09-14T08:54:47.817 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_8.status 2024-09-14T08:54:47.817 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_8 --format xml 2024-09-14T08:54:47.817 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:54:48.093 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:54:48.094 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=42 2024-09-14T08:54:48.095 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 100 = 42 2024-09-14T08:54:48.095 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:54:48.095 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1.6 2024-09-14T08:54:49.696 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_8 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_8.status 2024-09-14T08:54:49.696 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:54:49.696 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:49.696 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_8 2024-09-14T08:54:49.697 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_8.status 2024-09-14T08:54:49.697 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_8 --format xml 2024-09-14T08:54:49.697 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:54:49.751 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:54:49.753 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=42 2024-09-14T08:54:49.753 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 100 = 42 2024-09-14T08:54:49.753 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:54:49.753 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 2 2024-09-14T08:54:51.755 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_8 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_8.status 2024-09-14T08:54:51.755 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:54:51.755 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:51.755 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_8 2024-09-14T08:54:51.755 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_8.status 2024-09-14T08:54:51.755 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_8 --format xml 2024-09-14T08:54:51.755 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:54:51.810 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:54:51.812 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=42 2024-09-14T08:54:51.812 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 100 = 42 2024-09-14T08:54:51.812 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:54:51.812 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 2 2024-09-14T08:54:52.938 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.8585850467191886) > weight (0.25) 2024-09-14T08:54:52.938 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.1 2024-09-14T08:54:52.938 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Sent signal 15 2024-09-14T08:54:52.938 INFO:tasks.rbd_mirror_thrash:waiting for 9 secs before reviving daemons 2024-09-14T08:54:52.939 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi164.stderr:2024-09-14T08:54:52.939+0000 7f383f65e640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 (PID: 48280) UID: 1000 2024-09-14T08:54:53.814 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_8 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_8.status 2024-09-14T08:54:53.814 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:54:53.814 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:53.814 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_8 2024-09-14T08:54:53.814 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_8.status 2024-09-14T08:54:53.814 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_8 --format xml 2024-09-14T08:54:53.814 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:54:53.868 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:54:53.870 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=42 2024-09-14T08:54:53.870 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 100 = 42 2024-09-14T08:54:53.870 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:54:53.870 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 4 2024-09-14T08:54:54.151 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.0 2024-09-14T08:54:54.151 INFO:teuthology.orchestra.run:waiting for 600 2024-09-14T08:54:54.151 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.0 2024-09-14T08:54:54.152 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Restarting daemon 2024-09-14T08:54:54.152 DEBUG:teuthology.orchestra.run.smithi164:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 2024-09-14T08:54:54.155 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Started 2024-09-14T08:54:54.155 INFO:tasks.rbd_mirror_thrash:waiting for 71 secs before thrashing 2024-09-14T08:54:57.871 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_8 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_8.status 2024-09-14T08:54:57.872 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:54:57.872 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:54:57.872 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_8 2024-09-14T08:54:57.872 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_8.status 2024-09-14T08:54:57.872 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_8 --format xml 2024-09-14T08:54:57.872 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:54:57.925 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:54:57.927 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=42 2024-09-14T08:54:57.927 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 100 = 42 2024-09-14T08:54:57.927 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:54:57.927 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 4 2024-09-14T08:55:01.928 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_8 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_8.status 2024-09-14T08:55:01.929 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:55:01.929 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:55:01.929 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_8 2024-09-14T08:55:01.929 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_8.status 2024-09-14T08:55:01.929 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_8 --format xml 2024-09-14T08:55:01.929 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:55:01.939 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.1 2024-09-14T08:55:01.939 INFO:teuthology.orchestra.run:waiting for 600 2024-09-14T08:55:01.940 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.1 2024-09-14T08:55:01.940 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Restarting daemon 2024-09-14T08:55:01.940 DEBUG:teuthology.orchestra.run.smithi164:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 2024-09-14T08:55:01.942 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Started 2024-09-14T08:55:01.943 INFO:tasks.rbd_mirror_thrash:waiting for 96 secs before thrashing 2024-09-14T08:55:01.984 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:55:01.986 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=42 2024-09-14T08:55:01.986 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 100 = 42 2024-09-14T08:55:01.986 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:55:01.986 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 8 2024-09-14T08:55:09.988 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_8 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_8.status 2024-09-14T08:55:09.988 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:55:09.988 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:55:09.988 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_8 2024-09-14T08:55:09.988 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_8.status 2024-09-14T08:55:09.988 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_8 --format xml 2024-09-14T08:55:09.988 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:55:10.042 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:55:10.044 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=42 2024-09-14T08:55:10.044 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 100 = 42 2024-09-14T08:55:10.044 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:55:10.044 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 8 2024-09-14T08:55:18.046 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_8 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_8.status 2024-09-14T08:55:18.046 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:55:18.046 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:55:18.046 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_8 2024-09-14T08:55:18.046 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_8.status 2024-09-14T08:55:18.046 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_8 --format xml 2024-09-14T08:55:18.046 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:55:18.100 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:55:18.102 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=42 2024-09-14T08:55:18.102 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 100 = 42 2024-09-14T08:55:18.102 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:55:18.102 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 16 2024-09-14T08:55:34.104 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_8 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_8.status 2024-09-14T08:55:34.104 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:55:34.104 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:55:34.104 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_8 2024-09-14T08:55:34.104 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_8.status 2024-09-14T08:55:34.105 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_8 --format xml 2024-09-14T08:55:34.105 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:55:34.160 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:55:34.162 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=100 2024-09-14T08:55:34.162 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 100 = 100 2024-09-14T08:55:34.162 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:55:34.162 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snap_present cluster1 mirror image_8 snap 2024-09-14T08:55:34.163 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:55:34.163 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:55:34.163 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_8 2024-09-14T08:55:34.163 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:55:34.163 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:55:34.163 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T08:55:34.163 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:55:35.163 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 info mirror/image_8@snap 2024-09-14T08:55:35.212 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_8': 2024-09-14T08:55:35.212 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T08:55:35.212 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:55:35.212 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 2 2024-09-14T08:55:35.213 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 127e2f8643fc 2024-09-14T08:55:35.213 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.127e2f8643fc 2024-09-14T08:55:35.213 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:55:35.213 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering, non-primary 2024-09-14T08:55:35.213 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:55:35.213 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:55:35.213 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:46:17 2024 2024-09-14T08:55:35.213 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:46:17 2024 2024-09-14T08:55:35.213 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:46:17 2024 2024-09-14T08:55:35.213 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: protected: False 2024-09-14T08:55:35.213 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:55:35.213 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:55:35.213 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: 9b43aad5-e6f1-4a31-a25c-d7df22bd01da 2024-09-14T08:55:35.214 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: false 2024-09-14T08:55:35.216 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:55:35.217 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ compare_image_snaps mirror image_8 snap 2024-09-14T08:55:35.217 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:55:35.217 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_8 2024-09-14T08:55:35.217 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:55:35.217 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local ret=0 2024-09-14T08:55:35.217 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local rmt_export=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_8.export 2024-09-14T08:55:35.217 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local loc_export=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_8.export 2024-09-14T08:55:35.217 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_8.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_8.export 2024-09-14T08:55:35.217 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror export image_8@snap /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_8.export 2024-09-14T08:55:35.419 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T08:55:35.426 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 -p mirror export image_8@snap /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_8.export 2024-09-14T08:55:35.679 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T08:55:35.685 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cmp /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_8.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_8.export 2024-09-14T08:55:35.739 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_8.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_8.export 2024-09-14T08:55:35.769 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:55:35.769 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T08:55:35.769 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_9 2024-09-14T08:55:35.769 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_snap cluster2 mirror image_9 snap 2024-09-14T08:55:35.769 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:55:35.769 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:55:35.769 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_9 2024-09-14T08:55:35.769 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:55:35.770 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror snap create image_9@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T08:55:36.569 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_started cluster1 mirror image_9 2024-09-14T08:55:36.569 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:55:36.569 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:55:36.569 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_9 2024-09-14T08:55:36.569 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_state cluster1 mirror image_9 started 2024-09-14T08:55:36.569 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:55:36.569 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:55:36.570 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_9 2024-09-14T08:55:36.570 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=started 2024-09-14T08:55:36.570 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:55:36.570 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:55:36.570 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:55:37.570 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_replay_state cluster1 mirror image_9 started 2024-09-14T08:55:37.571 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:55:37.571 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:55:37.571 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_9 2024-09-14T08:55:37.571 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=started 2024-09-14T08:55:37.571 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_result 2024-09-14T08:55:37.571 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=stopped 2024-09-14T08:55:37.572 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_9 2024-09-14T08:55:37.572 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_instance=cluster1 2024-09-14T08:55:37.572 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:55:37.572 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T08:55:37.572 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ grep -i state 2024-09-14T08:55:37.572 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance=cluster1 2024-09-14T08:55:37.573 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local loop_instance 2024-09-14T08:55:37.573 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:55:37.573 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sleep 0 2024-09-14T08:55:37.573 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T08:55:37.576 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ seq 0 3 2024-09-14T08:55:37.577 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:55:37.577 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_9 2024-09-14T08:55:37.577 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:0 2024-09-14T08:55:37.577 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:55:37.577 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:55:37.577 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T08:55:37.577 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:0 2024-09-14T08:55:37.577 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:55:37.577 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:55:37.577 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:55:37.577 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=0 2024-09-14T08:55:37.577 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T08:55:37.578 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:55:37.578 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:55:37.578 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=0 2024-09-14T08:55:37.578 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:55:37.578 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T08:55:37.578 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:0 2024-09-14T08:55:37.578 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:55:37.578 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:55:37.578 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T08:55:37.578 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:0 2024-09-14T08:55:37.578 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:55:37.578 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:55:37.579 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:55:37.579 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:55:37.579 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T08:55:37.579 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:55:37.579 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:55:37.579 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=0 2024-09-14T08:55:37.579 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=0 2024-09-14T08:55:37.579 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T08:55:37.597 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:55:37.597 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:55:37.597 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:55:37.597 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_9 2024-09-14T08:55:37.686 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_9'] 2024-09-14T08:55:37.686 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:55:37.686 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:55:37.689 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:55:37.689 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_9 2024-09-14T08:55:37.689 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:1 2024-09-14T08:55:37.690 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:55:37.690 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:55:37.690 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T08:55:37.690 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:1 2024-09-14T08:55:37.690 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:55:37.690 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:55:37.690 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:55:37.690 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=1 2024-09-14T08:55:37.690 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T08:55:37.690 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:55:37.690 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:55:37.690 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=1 2024-09-14T08:55:37.690 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:55:37.690 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T08:55:37.690 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:1 2024-09-14T08:55:37.691 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:55:37.691 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:55:37.691 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T08:55:37.691 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:1 2024-09-14T08:55:37.691 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:55:37.691 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:55:37.691 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:55:37.691 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:55:37.691 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T08:55:37.691 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:55:37.691 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:55:37.691 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=1 2024-09-14T08:55:37.691 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=1 2024-09-14T08:55:37.691 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T08:55:37.709 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:55:37.709 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:55:37.709 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:55:37.709 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_9 2024-09-14T08:55:37.791 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_9'] 2024-09-14T08:55:37.792 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:55:37.792 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:55:37.795 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:55:37.795 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_9 2024-09-14T08:55:37.795 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:2 2024-09-14T08:55:37.795 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:55:37.795 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:55:37.795 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-14T08:55:37.795 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:2 2024-09-14T08:55:37.796 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:55:37.796 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:55:37.796 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:55:37.796 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=2 2024-09-14T08:55:37.796 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T08:55:37.796 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:55:37.796 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:55:37.796 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=2 2024-09-14T08:55:37.796 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:55:37.796 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-14T08:55:37.796 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:2 2024-09-14T08:55:37.796 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:55:37.796 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:55:37.796 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T08:55:37.796 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:2 2024-09-14T08:55:37.797 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:55:37.797 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:55:37.797 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:55:37.797 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:55:37.797 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-14T08:55:37.797 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:55:37.797 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:55:37.797 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=2 2024-09-14T08:55:37.797 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=2 2024-09-14T08:55:37.797 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T08:55:37.813 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:55:37.813 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:55:37.813 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:55:37.813 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_9 2024-09-14T08:55:37.902 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ return 0 2024-09-14T08:55:37.902 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T08:55:37.902 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ echo ' "state": "Replaying"' 2024-09-14T08:55:37.903 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep -i Replaying 2024-09-14T08:55:37.903 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: "state": "Replaying" 2024-09-14T08:55:37.905 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_state=started 2024-09-14T08:55:37.905 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test started = started 2024-09-14T08:55:37.906 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:55:37.906 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_9 2024-09-14T08:55:37.906 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:55:37.906 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:55:37.906 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:55:37.906 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_9 2024-09-14T08:55:37.906 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = journal ']' 2024-09-14T08:55:37.906 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = snapshot ']' 2024-09-14T08:55:37.906 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_9 2024-09-14T08:55:37.906 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:55:37.906 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:55:37.906 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:55:37.907 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_9 2024-09-14T08:55:37.907 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster2-mirror-image_9.status 2024-09-14T08:55:37.907 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster2-mirror-image_9.status 2024-09-14T08:55:37.907 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:55:37.907 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_9.status 2024-09-14T08:55:37.907 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster1-mirror-image_9.status 2024-09-14T08:55:37.908 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster1-mirror-image_9.status 2024-09-14T08:55:37.908 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:55:37.908 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_status_log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_9.status 2024-09-14T08:55:37.908 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ mirror_image_snapshot cluster2 mirror image_9 2024-09-14T08:55:37.908 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:55:37.908 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:55:37.908 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_9 2024-09-14T08:55:37.908 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_9 2024-09-14T08:55:38.577 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Snapshot ID: 102 2024-09-14T08:55:38.587 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_9 /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_9.status 2024-09-14T08:55:38.587 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:55:38.587 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:55:38.587 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_9 2024-09-14T08:55:38.587 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_9.status 2024-09-14T08:55:38.587 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_9 --format xml 2024-09-14T08:55:38.587 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:55:38.633 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-14T08:55:38.635 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snapshot_id=102 2024-09-14T08:55:38.635 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:55:38.635 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:55:38.635 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.2 2024-09-14T08:55:38.836 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_9 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_9.status 2024-09-14T08:55:38.836 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:55:38.836 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:55:38.836 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_9 2024-09-14T08:55:38.836 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_9.status 2024-09-14T08:55:38.836 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_9 --format xml 2024-09-14T08:55:38.837 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:55:38.884 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:55:38.886 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=43 2024-09-14T08:55:38.886 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 102 = 43 2024-09-14T08:55:38.886 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:55:38.886 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.4 2024-09-14T08:55:39.288 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_9 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_9.status 2024-09-14T08:55:39.288 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:55:39.288 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:55:39.288 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_9 2024-09-14T08:55:39.288 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_9.status 2024-09-14T08:55:39.288 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_9 --format xml 2024-09-14T08:55:39.288 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:55:39.336 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:55:39.338 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=43 2024-09-14T08:55:39.338 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 102 = 43 2024-09-14T08:55:39.338 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:55:39.338 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.8 2024-09-14T08:55:40.139 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_9 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_9.status 2024-09-14T08:55:40.140 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:55:40.140 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:55:40.140 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_9 2024-09-14T08:55:40.140 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_9.status 2024-09-14T08:55:40.140 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_9 --format xml 2024-09-14T08:55:40.140 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:55:40.190 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:55:40.190 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:55:40.192 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:55:40.192 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:55:40.192 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:55:40.193 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:55:40.193 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 102 = '' 2024-09-14T08:55:40.193 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:55:40.193 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1.6 2024-09-14T08:55:41.794 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_9 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_9.status 2024-09-14T08:55:41.794 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:55:41.794 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:55:41.794 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_9 2024-09-14T08:55:41.794 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_9.status 2024-09-14T08:55:41.794 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_9 --format xml 2024-09-14T08:55:41.795 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:55:41.846 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:55:41.848 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=102 2024-09-14T08:55:41.849 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 102 = 102 2024-09-14T08:55:41.849 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:55:41.849 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snap_present cluster1 mirror image_9 snap 2024-09-14T08:55:41.849 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:55:41.849 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:55:41.849 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_9 2024-09-14T08:55:41.849 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:55:41.849 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:55:41.849 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T08:55:41.849 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:55:42.849 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 info mirror/image_9@snap 2024-09-14T08:55:42.898 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_9': 2024-09-14T08:55:42.899 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T08:55:42.899 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:55:42.899 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 2 2024-09-14T08:55:42.899 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 10bdaf30d59d 2024-09-14T08:55:42.899 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.10bdaf30d59d 2024-09-14T08:55:42.899 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:55:42.899 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering, non-primary 2024-09-14T08:55:42.899 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:55:42.899 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:55:42.899 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:46:24 2024 2024-09-14T08:55:42.899 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:46:24 2024 2024-09-14T08:55:42.899 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:46:24 2024 2024-09-14T08:55:42.899 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: protected: False 2024-09-14T08:55:42.900 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:55:42.900 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:55:42.900 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: 6b16cbe5-ac26-4754-a383-29c75eb5ef3c 2024-09-14T08:55:42.900 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: false 2024-09-14T08:55:42.903 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:55:42.903 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ compare_image_snaps mirror image_9 snap 2024-09-14T08:55:42.903 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:55:42.903 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_9 2024-09-14T08:55:42.903 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:55:42.903 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local ret=0 2024-09-14T08:55:42.903 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local rmt_export=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_9.export 2024-09-14T08:55:42.903 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local loc_export=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_9.export 2024-09-14T08:55:42.903 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_9.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_9.export 2024-09-14T08:55:42.904 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror export image_9@snap /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_9.export 2024-09-14T08:55:43.102 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T08:55:43.109 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 -p mirror export image_9@snap /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_9.export 2024-09-14T08:55:43.321 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T08:55:43.327 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cmp /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_9.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_9.export 2024-09-14T08:55:43.382 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_9.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_9.export 2024-09-14T08:55:43.413 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:55:43.413 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T08:55:43.413 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_10 2024-09-14T08:55:43.413 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_snap cluster2 mirror image_10 snap 2024-09-14T08:55:43.414 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:55:43.414 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:55:43.414 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_10 2024-09-14T08:55:43.414 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:55:43.414 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror snap create image_10@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T08:55:43.580 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_started cluster1 mirror image_10 2024-09-14T08:55:43.581 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:55:43.581 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:55:43.581 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_10 2024-09-14T08:55:43.581 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_state cluster1 mirror image_10 started 2024-09-14T08:55:43.581 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:55:43.581 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:55:43.581 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_10 2024-09-14T08:55:43.581 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=started 2024-09-14T08:55:43.581 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:55:43.581 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:55:43.581 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:55:44.582 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_replay_state cluster1 mirror image_10 started 2024-09-14T08:55:44.582 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:55:44.582 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:55:44.582 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_10 2024-09-14T08:55:44.582 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=started 2024-09-14T08:55:44.583 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_result 2024-09-14T08:55:44.583 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=stopped 2024-09-14T08:55:44.583 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_10 2024-09-14T08:55:44.583 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_instance=cluster1 2024-09-14T08:55:44.583 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:55:44.583 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T08:55:44.583 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ grep -i state 2024-09-14T08:55:44.583 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance=cluster1 2024-09-14T08:55:44.583 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local loop_instance 2024-09-14T08:55:44.583 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:55:44.583 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sleep 0 2024-09-14T08:55:44.586 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T08:55:44.587 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ seq 0 3 2024-09-14T08:55:44.587 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:55:44.587 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_10 2024-09-14T08:55:44.587 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:0 2024-09-14T08:55:44.587 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:55:44.587 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:55:44.587 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T08:55:44.587 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:0 2024-09-14T08:55:44.587 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:55:44.587 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:55:44.587 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:55:44.587 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=0 2024-09-14T08:55:44.588 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T08:55:44.588 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:55:44.588 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:55:44.588 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=0 2024-09-14T08:55:44.588 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:55:44.588 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T08:55:44.588 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:0 2024-09-14T08:55:44.588 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:55:44.588 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:55:44.588 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T08:55:44.588 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:0 2024-09-14T08:55:44.588 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:55:44.588 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:55:44.589 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:55:44.589 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:55:44.589 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T08:55:44.589 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:55:44.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:55:44.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=0 2024-09-14T08:55:44.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=0 2024-09-14T08:55:44.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T08:55:44.607 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:55:44.607 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:55:44.608 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:55:44.608 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_10 2024-09-14T08:55:44.701 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ return 0 2024-09-14T08:55:44.702 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T08:55:44.702 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ echo ' "state": "Replaying"' 2024-09-14T08:55:44.702 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep -i Replaying 2024-09-14T08:55:44.703 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: "state": "Replaying" 2024-09-14T08:55:44.704 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_state=started 2024-09-14T08:55:44.705 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test started = started 2024-09-14T08:55:44.705 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:55:44.705 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_10 2024-09-14T08:55:44.705 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:55:44.705 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:55:44.705 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:55:44.705 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_10 2024-09-14T08:55:44.705 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = journal ']' 2024-09-14T08:55:44.705 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = snapshot ']' 2024-09-14T08:55:44.705 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_10 2024-09-14T08:55:44.705 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:55:44.705 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:55:44.705 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:55:44.706 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_10 2024-09-14T08:55:44.706 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster2-mirror-image_10.status 2024-09-14T08:55:44.706 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster2-mirror-image_10.status 2024-09-14T08:55:44.706 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:55:44.708 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_10.status 2024-09-14T08:55:44.708 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster1-mirror-image_10.status 2024-09-14T08:55:44.708 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster1-mirror-image_10.status 2024-09-14T08:55:44.708 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:55:44.709 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_status_log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_10.status 2024-09-14T08:55:44.709 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ mirror_image_snapshot cluster2 mirror image_10 2024-09-14T08:55:44.709 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:55:44.709 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:55:44.709 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_10 2024-09-14T08:55:44.709 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_10 2024-09-14T08:55:45.598 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Snapshot ID: 104 2024-09-14T08:55:45.605 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_10 /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_10.status 2024-09-14T08:55:45.605 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:55:45.605 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:55:45.605 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_10 2024-09-14T08:55:45.605 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_10.status 2024-09-14T08:55:45.605 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_10 --format xml 2024-09-14T08:55:45.605 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:55:45.655 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-14T08:55:45.657 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snapshot_id=104 2024-09-14T08:55:45.657 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:55:45.657 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:55:45.657 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.2 2024-09-14T08:55:45.858 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_10 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_10.status 2024-09-14T08:55:45.858 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:55:45.858 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:55:45.858 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_10 2024-09-14T08:55:45.858 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_10.status 2024-09-14T08:55:45.859 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_10 --format xml 2024-09-14T08:55:45.859 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:55:45.912 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:55:45.914 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=44 2024-09-14T08:55:45.914 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 104 = 44 2024-09-14T08:55:45.914 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:55:45.914 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.4 2024-09-14T08:55:46.315 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_10 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_10.status 2024-09-14T08:55:46.316 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:55:46.316 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:55:46.316 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_10 2024-09-14T08:55:46.316 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_10.status 2024-09-14T08:55:46.316 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_10 --format xml 2024-09-14T08:55:46.316 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:55:46.369 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:55:46.371 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=44 2024-09-14T08:55:46.371 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 104 = 44 2024-09-14T08:55:46.371 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:55:46.371 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.8 2024-09-14T08:55:47.173 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_10 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_10.status 2024-09-14T08:55:47.173 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:55:47.173 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:55:47.173 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_10 2024-09-14T08:55:47.173 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_10.status 2024-09-14T08:55:47.173 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_10 --format xml 2024-09-14T08:55:47.173 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:55:47.228 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:55:47.228 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:55:47.229 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:55:47.229 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:55:47.230 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:55:47.230 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:55:47.230 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 104 = '' 2024-09-14T08:55:47.230 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:55:47.230 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1.6 2024-09-14T08:55:48.832 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_10 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_10.status 2024-09-14T08:55:48.832 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:55:48.832 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:55:48.832 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_10 2024-09-14T08:55:48.832 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_10.status 2024-09-14T08:55:48.832 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_10 --format xml 2024-09-14T08:55:48.832 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:55:48.882 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:55:48.885 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=104 2024-09-14T08:55:48.885 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 104 = 104 2024-09-14T08:55:48.885 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:55:48.885 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snap_present cluster1 mirror image_10 snap 2024-09-14T08:55:48.885 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:55:48.885 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:55:48.886 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_10 2024-09-14T08:55:48.886 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:55:48.886 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:55:48.886 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T08:55:48.886 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:55:49.885 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 info mirror/image_10@snap 2024-09-14T08:55:49.933 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_10': 2024-09-14T08:55:49.933 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T08:55:49.933 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:55:49.933 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 2 2024-09-14T08:55:49.933 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 10c2f1ac420d 2024-09-14T08:55:49.933 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.10c2f1ac420d 2024-09-14T08:55:49.933 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:55:49.933 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering, non-primary 2024-09-14T08:55:49.933 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:55:49.933 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:55:49.933 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:46:31 2024 2024-09-14T08:55:49.933 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:46:31 2024 2024-09-14T08:55:49.934 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:46:31 2024 2024-09-14T08:55:49.934 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: protected: False 2024-09-14T08:55:49.934 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:55:49.934 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:55:49.934 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: 0a80ae74-9f85-4745-bf89-a713c5afc298 2024-09-14T08:55:49.934 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: false 2024-09-14T08:55:49.937 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:55:49.937 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ compare_image_snaps mirror image_10 snap 2024-09-14T08:55:49.937 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:55:49.938 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_10 2024-09-14T08:55:49.938 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:55:49.938 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local ret=0 2024-09-14T08:55:49.938 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local rmt_export=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_10.export 2024-09-14T08:55:49.938 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local loc_export=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_10.export 2024-09-14T08:55:49.938 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_10.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_10.export 2024-09-14T08:55:49.938 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror export image_10@snap /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_10.export 2024-09-14T08:55:50.148 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T08:55:50.155 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 -p mirror export image_10@snap /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_10.export 2024-09-14T08:55:50.414 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T08:55:50.421 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cmp /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_10.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_10.export 2024-09-14T08:55:50.476 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_10.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_10.export 2024-09-14T08:55:50.506 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:55:50.507 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T08:55:50.507 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_11 2024-09-14T08:55:50.507 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_snap cluster2 mirror image_11 snap 2024-09-14T08:55:50.507 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:55:50.507 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:55:50.507 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_11 2024-09-14T08:55:50.507 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:55:50.507 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror snap create image_11@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T08:55:50.594 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_started cluster1 mirror image_11 2024-09-14T08:55:50.594 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:55:50.594 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:55:50.594 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_11 2024-09-14T08:55:50.594 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_state cluster1 mirror image_11 started 2024-09-14T08:55:50.595 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:55:50.595 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:55:50.595 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_11 2024-09-14T08:55:50.595 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=started 2024-09-14T08:55:50.595 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:55:50.595 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:55:50.595 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:55:51.595 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_replay_state cluster1 mirror image_11 started 2024-09-14T08:55:51.595 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:55:51.596 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:55:51.596 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_11 2024-09-14T08:55:51.596 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=started 2024-09-14T08:55:51.596 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_result 2024-09-14T08:55:51.596 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=stopped 2024-09-14T08:55:51.597 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_11 2024-09-14T08:55:51.597 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_instance=cluster1 2024-09-14T08:55:51.597 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ grep -i state 2024-09-14T08:55:51.597 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:55:51.597 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T08:55:51.597 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance=cluster1 2024-09-14T08:55:51.597 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local loop_instance 2024-09-14T08:55:51.597 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:55:51.598 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sleep 0 2024-09-14T08:55:51.598 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T08:55:51.598 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ seq 0 3 2024-09-14T08:55:51.600 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:55:51.600 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_11 2024-09-14T08:55:51.600 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:0 2024-09-14T08:55:51.600 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:55:51.601 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:55:51.601 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T08:55:51.601 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:0 2024-09-14T08:55:51.601 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:55:51.601 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:55:51.601 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:55:51.601 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=0 2024-09-14T08:55:51.601 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T08:55:51.601 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:55:51.601 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:55:51.601 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=0 2024-09-14T08:55:51.601 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:55:51.601 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T08:55:51.602 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:0 2024-09-14T08:55:51.602 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:55:51.602 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:55:51.602 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T08:55:51.602 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:0 2024-09-14T08:55:51.602 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:55:51.602 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:55:51.602 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:55:51.602 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:55:51.602 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T08:55:51.602 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:55:51.602 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:55:51.602 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=0 2024-09-14T08:55:51.602 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=0 2024-09-14T08:55:51.602 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T08:55:51.619 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:55:51.619 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:55:51.619 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:55:51.619 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_11 2024-09-14T08:55:51.702 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_11'] 2024-09-14T08:55:51.702 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:55:51.703 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:55:51.705 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:55:51.706 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_11 2024-09-14T08:55:51.706 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:1 2024-09-14T08:55:51.706 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:55:51.706 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:55:51.706 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T08:55:51.706 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:1 2024-09-14T08:55:51.706 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:55:51.706 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:55:51.706 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:55:51.706 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=1 2024-09-14T08:55:51.706 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T08:55:51.706 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:55:51.706 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:55:51.707 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=1 2024-09-14T08:55:51.707 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:55:51.707 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T08:55:51.707 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:1 2024-09-14T08:55:51.707 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:55:51.707 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:55:51.707 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T08:55:51.707 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:1 2024-09-14T08:55:51.707 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:55:51.707 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:55:51.707 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:55:51.707 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:55:51.707 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T08:55:51.707 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:55:51.707 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:55:51.708 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=1 2024-09-14T08:55:51.708 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=1 2024-09-14T08:55:51.708 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T08:55:51.725 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:55:51.725 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:55:51.725 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:55:51.725 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_11 2024-09-14T08:55:51.807 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_11'] 2024-09-14T08:55:51.808 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:55:51.808 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:55:51.811 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:55:51.811 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_11 2024-09-14T08:55:51.811 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:2 2024-09-14T08:55:51.811 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:55:51.811 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:55:51.811 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-14T08:55:51.811 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:2 2024-09-14T08:55:51.811 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:55:51.811 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:55:51.811 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:55:51.811 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=2 2024-09-14T08:55:51.812 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T08:55:51.812 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:55:51.812 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:55:51.812 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=2 2024-09-14T08:55:51.812 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:55:51.812 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-14T08:55:51.812 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:2 2024-09-14T08:55:51.812 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:55:51.812 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:55:51.812 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T08:55:51.812 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:2 2024-09-14T08:55:51.812 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:55:51.812 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:55:51.813 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:55:51.813 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:55:51.813 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-14T08:55:51.813 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:55:51.813 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:55:51.813 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=2 2024-09-14T08:55:51.813 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=2 2024-09-14T08:55:51.813 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T08:55:51.829 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:55:51.829 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:55:51.829 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:55:51.829 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_11 2024-09-14T08:55:51.911 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_11'] 2024-09-14T08:55:51.911 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:55:51.911 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:55:51.914 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:55:51.914 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_11 2024-09-14T08:55:51.915 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:3 2024-09-14T08:55:51.915 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:55:51.915 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:55:51.915 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:3 cluster instance 2024-09-14T08:55:51.915 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:3 2024-09-14T08:55:51.915 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:55:51.915 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:55:51.915 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:55:51.915 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=3 2024-09-14T08:55:51.915 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-14T08:55:51.915 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:55:51.915 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:55:51.915 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=3 2024-09-14T08:55:51.915 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=3 2024-09-14T08:55:51.916 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:3 cluster1 2024-09-14T08:55:51.916 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:3 2024-09-14T08:55:51.916 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:55:51.916 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:55:51.916 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-14T08:55:51.916 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:3 2024-09-14T08:55:51.916 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:55:51.916 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:55:51.916 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:55:51.916 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=3 2024-09-14T08:55:51.916 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 3 = cluster1:3 ']' 2024-09-14T08:55:51.916 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:55:51.916 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:55:51.916 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=3 2024-09-14T08:55:51.917 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=3 2024-09-14T08:55:51.917 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-14T08:55:51.932 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T08:55:51.932 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T08:55:51.933 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T08:55:51.933 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_11 2024-09-14T08:55:52.022 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ return 0 2024-09-14T08:55:52.023 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T08:55:52.023 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ echo ' "state": "Replaying"' 2024-09-14T08:55:52.023 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep -i Replaying 2024-09-14T08:55:52.024 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: "state": "Replaying" 2024-09-14T08:55:52.027 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_state=started 2024-09-14T08:55:52.027 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test started = started 2024-09-14T08:55:52.027 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:55:52.027 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_11 2024-09-14T08:55:52.028 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:55:52.028 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:55:52.028 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:55:52.028 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_11 2024-09-14T08:55:52.028 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = journal ']' 2024-09-14T08:55:52.028 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = snapshot ']' 2024-09-14T08:55:52.029 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_11 2024-09-14T08:55:52.029 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:55:52.029 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:55:52.029 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:55:52.029 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_11 2024-09-14T08:55:52.030 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster2-mirror-image_11.status 2024-09-14T08:55:52.030 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster2-mirror-image_11.status 2024-09-14T08:55:52.030 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:55:52.030 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_11.status 2024-09-14T08:55:52.030 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster1-mirror-image_11.status 2024-09-14T08:55:52.030 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster1-mirror-image_11.status 2024-09-14T08:55:52.030 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:55:52.032 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_status_log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_11.status 2024-09-14T08:55:52.032 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ mirror_image_snapshot cluster2 mirror image_11 2024-09-14T08:55:52.032 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:55:52.032 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:55:52.032 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_11 2024-09-14T08:55:52.032 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_11 2024-09-14T08:55:52.608 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Snapshot ID: 106 2024-09-14T08:55:52.614 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_11 /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_11.status 2024-09-14T08:55:52.615 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:55:52.615 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:55:52.615 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_11 2024-09-14T08:55:52.615 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_11.status 2024-09-14T08:55:52.615 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_11 --format xml 2024-09-14T08:55:52.615 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:55:52.662 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-14T08:55:52.664 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snapshot_id=106 2024-09-14T08:55:52.664 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:55:52.664 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:55:52.664 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.2 2024-09-14T08:55:52.865 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_11 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_11.status 2024-09-14T08:55:52.866 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:55:52.866 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:55:52.866 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_11 2024-09-14T08:55:52.866 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_11.status 2024-09-14T08:55:52.866 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_11 --format xml 2024-09-14T08:55:52.867 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:55:52.915 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:55:52.917 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=45 2024-09-14T08:55:52.917 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 106 = 45 2024-09-14T08:55:52.917 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:55:52.917 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.4 2024-09-14T08:55:53.318 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_11 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_11.status 2024-09-14T08:55:53.318 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:55:53.318 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:55:53.318 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_11 2024-09-14T08:55:53.318 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_11.status 2024-09-14T08:55:53.319 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_11 --format xml 2024-09-14T08:55:53.319 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:55:53.368 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:55:53.370 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=45 2024-09-14T08:55:53.370 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 106 = 45 2024-09-14T08:55:53.370 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:55:53.370 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.8 2024-09-14T08:55:54.171 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_11 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_11.status 2024-09-14T08:55:54.172 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:55:54.172 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:55:54.172 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_11 2024-09-14T08:55:54.172 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_11.status 2024-09-14T08:55:54.172 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_11 --format xml 2024-09-14T08:55:54.172 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:55:54.224 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:55:54.224 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:55:54.225 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:55:54.225 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:55:54.225 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:55:54.226 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:55:54.226 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 106 = '' 2024-09-14T08:55:54.226 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:55:54.226 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1.6 2024-09-14T08:55:55.827 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_11 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_11.status 2024-09-14T08:55:55.827 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:55:55.828 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:55:55.828 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_11 2024-09-14T08:55:55.828 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_11.status 2024-09-14T08:55:55.828 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_11 --format xml 2024-09-14T08:55:55.828 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:55:55.877 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:55:55.879 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=106 2024-09-14T08:55:55.879 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 106 = 106 2024-09-14T08:55:55.879 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:55:55.880 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snap_present cluster1 mirror image_11 snap 2024-09-14T08:55:55.880 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:55:55.880 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:55:55.880 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_11 2024-09-14T08:55:55.880 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:55:55.880 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:55:55.880 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T08:55:55.880 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:55:56.880 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 info mirror/image_11@snap 2024-09-14T08:55:56.930 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_11': 2024-09-14T08:55:56.930 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T08:55:56.930 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:55:56.930 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 2 2024-09-14T08:55:56.930 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 1189b1511c07 2024-09-14T08:55:56.930 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.1189b1511c07 2024-09-14T08:55:56.930 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:55:56.930 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering, non-primary 2024-09-14T08:55:56.931 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:55:56.931 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:55:56.931 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:46:46 2024 2024-09-14T08:55:56.931 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:46:46 2024 2024-09-14T08:55:56.931 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:46:46 2024 2024-09-14T08:55:56.931 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: protected: False 2024-09-14T08:55:56.931 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:55:56.931 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:55:56.931 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: f2624247-ccb3-494f-a736-10ddf00ce6ae 2024-09-14T08:55:56.931 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: false 2024-09-14T08:55:56.934 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:55:56.934 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ compare_image_snaps mirror image_11 snap 2024-09-14T08:55:56.934 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:55:56.934 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_11 2024-09-14T08:55:56.935 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:55:56.935 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local ret=0 2024-09-14T08:55:56.935 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local rmt_export=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_11.export 2024-09-14T08:55:56.935 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local loc_export=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_11.export 2024-09-14T08:55:56.935 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_11.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_11.export 2024-09-14T08:55:56.935 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror export image_11@snap /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_11.export 2024-09-14T08:55:57.145 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T08:55:57.153 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 -p mirror export image_11@snap /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_11.export 2024-09-14T08:55:57.404 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T08:55:57.411 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cmp /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_11.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_11.export 2024-09-14T08:55:57.465 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_11.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_11.export 2024-09-14T08:55:57.497 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:55:57.497 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T08:55:57.497 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_12 2024-09-14T08:55:57.497 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_snap cluster2 mirror image_12 snap 2024-09-14T08:55:57.497 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:55:57.497 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:55:57.497 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_12 2024-09-14T08:55:57.497 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:55:57.498 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror snap create image_12@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T08:55:57.619 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_started cluster1 mirror image_12 2024-09-14T08:55:57.620 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:55:57.620 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:55:57.620 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_12 2024-09-14T08:55:57.620 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_state cluster1 mirror image_12 started 2024-09-14T08:55:57.620 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:55:57.620 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:55:57.620 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_12 2024-09-14T08:55:57.620 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=started 2024-09-14T08:55:57.620 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:55:57.620 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:55:57.620 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:55:58.621 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_replay_state cluster1 mirror image_12 started 2024-09-14T08:55:58.621 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:55:58.621 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:55:58.621 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_12 2024-09-14T08:55:58.621 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=started 2024-09-14T08:55:58.621 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_result 2024-09-14T08:55:58.622 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=stopped 2024-09-14T08:55:58.623 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_12 2024-09-14T08:55:58.623 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_instance=cluster1 2024-09-14T08:55:58.623 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ grep -i state 2024-09-14T08:55:58.623 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:55:58.623 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T08:55:58.623 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance=cluster1 2024-09-14T08:55:58.624 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local loop_instance 2024-09-14T08:55:58.624 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:55:58.624 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sleep 0 2024-09-14T08:55:58.624 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T08:55:58.624 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ seq 0 3 2024-09-14T08:55:58.626 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:55:58.626 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_12 2024-09-14T08:55:58.626 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:0 2024-09-14T08:55:58.626 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:55:58.626 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:55:58.626 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T08:55:58.626 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:0 2024-09-14T08:55:58.626 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:55:58.626 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:55:58.626 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:55:58.626 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=0 2024-09-14T08:55:58.626 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T08:55:58.627 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:55:58.627 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:55:58.627 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=0 2024-09-14T08:55:58.627 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:55:58.627 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T08:55:58.627 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:0 2024-09-14T08:55:58.627 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:55:58.627 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:55:58.627 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T08:55:58.629 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:0 2024-09-14T08:55:58.629 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:55:58.629 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:55:58.629 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:55:58.629 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:55:58.629 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T08:55:58.629 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:55:58.629 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:55:58.629 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=0 2024-09-14T08:55:58.629 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=0 2024-09-14T08:55:58.630 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T08:55:58.647 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:55:58.647 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:55:58.647 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:55:58.647 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_12 2024-09-14T08:55:58.735 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_12'] 2024-09-14T08:55:58.736 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:55:58.736 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:55:58.739 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:55:58.739 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_12 2024-09-14T08:55:58.739 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:1 2024-09-14T08:55:58.739 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:55:58.739 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:55:58.739 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T08:55:58.739 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:1 2024-09-14T08:55:58.739 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:55:58.739 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:55:58.739 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:55:58.739 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=1 2024-09-14T08:55:58.739 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T08:55:58.739 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:55:58.740 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:55:58.740 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=1 2024-09-14T08:55:58.740 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:55:58.740 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T08:55:58.740 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:1 2024-09-14T08:55:58.740 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:55:58.740 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:55:58.740 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T08:55:58.740 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:1 2024-09-14T08:55:58.740 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:55:58.741 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:55:58.741 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:55:58.741 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:55:58.741 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T08:55:58.741 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:55:58.741 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:55:58.741 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=1 2024-09-14T08:55:58.741 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=1 2024-09-14T08:55:58.741 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T08:55:58.759 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:55:58.759 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:55:58.759 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:55:58.759 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_12 2024-09-14T08:55:58.848 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ return 0 2024-09-14T08:55:58.848 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T08:55:58.849 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ echo ' "state": "Replaying"' 2024-09-14T08:55:58.849 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep -i Replaying 2024-09-14T08:55:58.849 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: "state": "Replaying" 2024-09-14T08:55:58.851 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_state=started 2024-09-14T08:55:58.851 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test started = started 2024-09-14T08:55:58.851 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:55:58.851 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_12 2024-09-14T08:55:58.851 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:55:58.851 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:55:58.851 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:55:58.852 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_12 2024-09-14T08:55:58.852 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = journal ']' 2024-09-14T08:55:58.852 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = snapshot ']' 2024-09-14T08:55:58.852 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_12 2024-09-14T08:55:58.852 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:55:58.852 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:55:58.852 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:55:58.852 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_12 2024-09-14T08:55:58.852 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster2-mirror-image_12.status 2024-09-14T08:55:58.852 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster2-mirror-image_12.status 2024-09-14T08:55:58.852 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:55:58.853 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_12.status 2024-09-14T08:55:58.853 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster1-mirror-image_12.status 2024-09-14T08:55:58.853 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster1-mirror-image_12.status 2024-09-14T08:55:58.854 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:55:58.854 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_status_log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_12.status 2024-09-14T08:55:58.854 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ mirror_image_snapshot cluster2 mirror image_12 2024-09-14T08:55:58.854 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:55:58.854 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:55:58.854 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_12 2024-09-14T08:55:58.854 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_12 2024-09-14T08:55:59.629 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Snapshot ID: 108 2024-09-14T08:55:59.635 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_12 /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_12.status 2024-09-14T08:55:59.635 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:55:59.636 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:55:59.636 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_12 2024-09-14T08:55:59.636 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_12.status 2024-09-14T08:55:59.636 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_12 --format xml 2024-09-14T08:55:59.636 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:55:59.687 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-14T08:55:59.689 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snapshot_id=108 2024-09-14T08:55:59.689 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:55:59.689 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:55:59.689 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.2 2024-09-14T08:55:59.890 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_12 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_12.status 2024-09-14T08:55:59.890 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:55:59.890 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:55:59.890 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_12 2024-09-14T08:55:59.890 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_12.status 2024-09-14T08:55:59.890 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_12 --format xml 2024-09-14T08:55:59.890 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:55:59.941 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:55:59.943 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=46 2024-09-14T08:55:59.943 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 108 = 46 2024-09-14T08:55:59.943 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:55:59.943 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.4 2024-09-14T08:56:00.344 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_12 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_12.status 2024-09-14T08:56:00.344 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:56:00.344 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:00.345 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_12 2024-09-14T08:56:00.345 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_12.status 2024-09-14T08:56:00.345 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_12 --format xml 2024-09-14T08:56:00.345 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:56:00.395 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:56:00.396 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=46 2024-09-14T08:56:00.397 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 108 = 46 2024-09-14T08:56:00.397 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:56:00.397 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.8 2024-09-14T08:56:01.198 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_12 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_12.status 2024-09-14T08:56:01.198 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:56:01.198 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:01.198 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_12 2024-09-14T08:56:01.198 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_12.status 2024-09-14T08:56:01.198 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_12 --format xml 2024-09-14T08:56:01.199 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:56:01.249 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:56:01.249 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:56:01.251 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:56:01.251 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:56:01.251 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:56:01.251 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:56:01.251 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 108 = '' 2024-09-14T08:56:01.251 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:56:01.251 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1.6 2024-09-14T08:56:02.852 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_12 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_12.status 2024-09-14T08:56:02.853 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:56:02.853 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:02.853 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_12 2024-09-14T08:56:02.853 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_12.status 2024-09-14T08:56:02.853 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_12 --format xml 2024-09-14T08:56:02.853 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:56:02.905 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:56:02.907 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=108 2024-09-14T08:56:02.907 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 108 = 108 2024-09-14T08:56:02.907 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:56:02.907 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snap_present cluster1 mirror image_12 snap 2024-09-14T08:56:02.907 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:56:02.907 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:02.907 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_12 2024-09-14T08:56:02.907 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:56:02.907 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:56:02.907 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T08:56:02.907 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:56:03.908 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 info mirror/image_12@snap 2024-09-14T08:56:03.957 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_12': 2024-09-14T08:56:03.958 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T08:56:03.958 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:56:03.958 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 2 2024-09-14T08:56:03.958 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 127e9c50154b 2024-09-14T08:56:03.958 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.127e9c50154b 2024-09-14T08:56:03.958 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:56:03.958 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering, non-primary 2024-09-14T08:56:03.958 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:56:03.958 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:56:03.958 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:46:55 2024 2024-09-14T08:56:03.958 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:46:55 2024 2024-09-14T08:56:03.958 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:46:55 2024 2024-09-14T08:56:03.958 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: protected: False 2024-09-14T08:56:03.959 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:56:03.959 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:56:03.959 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: a4a9b638-bea9-4763-b984-2d72d49c1189 2024-09-14T08:56:03.959 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: false 2024-09-14T08:56:03.962 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:56:03.962 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ compare_image_snaps mirror image_12 snap 2024-09-14T08:56:03.962 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:03.962 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_12 2024-09-14T08:56:03.962 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:56:03.962 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local ret=0 2024-09-14T08:56:03.962 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local rmt_export=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_12.export 2024-09-14T08:56:03.962 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local loc_export=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_12.export 2024-09-14T08:56:03.962 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_12.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_12.export 2024-09-14T08:56:03.963 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror export image_12@snap /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_12.export 2024-09-14T08:56:04.158 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T08:56:04.165 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 -p mirror export image_12@snap /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_12.export 2024-09-14T08:56:04.421 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T08:56:04.426 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cmp /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_12.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_12.export 2024-09-14T08:56:04.480 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_12.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_12.export 2024-09-14T08:56:04.512 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:56:04.512 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T08:56:04.512 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_13 2024-09-14T08:56:04.512 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_snap cluster2 mirror image_13 snap 2024-09-14T08:56:04.512 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:56:04.512 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:04.512 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_13 2024-09-14T08:56:04.512 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:56:04.513 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror snap create image_13@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T08:56:04.633 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_started cluster1 mirror image_13 2024-09-14T08:56:04.634 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:56:04.634 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:04.634 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_13 2024-09-14T08:56:04.634 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_state cluster1 mirror image_13 started 2024-09-14T08:56:04.634 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:56:04.634 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:04.634 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_13 2024-09-14T08:56:04.634 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=started 2024-09-14T08:56:04.634 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:56:04.634 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:56:04.634 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:56:05.157 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.0 2024-09-14T08:56:05.157 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Sent signal 15 2024-09-14T08:56:05.158 INFO:tasks.rbd_mirror_thrash:waiting for 1 secs before reviving daemons 2024-09-14T08:56:05.158 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi164.stderr:2024-09-14T08:56:05.158+0000 7f768b104640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 (PID: 52036) UID: 1000 2024-09-14T08:56:05.636 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_replay_state cluster1 mirror image_13 started 2024-09-14T08:56:05.636 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:56:05.636 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:05.636 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_13 2024-09-14T08:56:05.637 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=started 2024-09-14T08:56:05.637 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_result 2024-09-14T08:56:05.637 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=stopped 2024-09-14T08:56:05.637 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_13 2024-09-14T08:56:05.637 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_instance=cluster1 2024-09-14T08:56:05.638 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:56:05.638 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ grep -i state 2024-09-14T08:56:05.638 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T08:56:05.638 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance=cluster1 2024-09-14T08:56:05.638 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local loop_instance 2024-09-14T08:56:05.638 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:56:05.638 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sleep 0 2024-09-14T08:56:05.643 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T08:56:05.643 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ seq 0 3 2024-09-14T08:56:05.643 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:56:05.643 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_13 2024-09-14T08:56:05.643 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:0 2024-09-14T08:56:05.643 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:56:05.644 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:56:05.644 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T08:56:05.644 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:0 2024-09-14T08:56:05.644 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:56:05.644 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:56:05.644 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:56:05.644 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=0 2024-09-14T08:56:05.644 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T08:56:05.644 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:56:05.644 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:56:05.644 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=0 2024-09-14T08:56:05.644 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:56:05.644 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T08:56:05.645 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:0 2024-09-14T08:56:05.645 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:56:05.645 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:56:05.645 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T08:56:05.645 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:0 2024-09-14T08:56:05.645 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:56:05.645 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:56:05.645 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:56:05.645 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:56:05.645 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T08:56:05.645 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:56:05.645 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:56:05.646 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=0 2024-09-14T08:56:05.646 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=0 2024-09-14T08:56:05.646 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T08:56:05.662 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:56:05.662 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:56:05.662 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:56:05.662 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_13 2024-09-14T08:56:05.732 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2024-09-14T08:56:05.735 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:56:05.735 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_13 2024-09-14T08:56:05.736 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:1 2024-09-14T08:56:05.736 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:56:05.736 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:56:05.736 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T08:56:05.736 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:1 2024-09-14T08:56:05.736 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:56:05.736 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:56:05.736 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:56:05.736 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=1 2024-09-14T08:56:05.736 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T08:56:05.736 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:56:05.736 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:56:05.736 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=1 2024-09-14T08:56:05.737 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:56:05.737 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T08:56:05.737 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:1 2024-09-14T08:56:05.737 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:56:05.737 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:56:05.737 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T08:56:05.737 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:1 2024-09-14T08:56:05.737 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:56:05.737 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:56:05.737 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:56:05.737 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:56:05.737 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T08:56:05.737 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:56:05.737 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:56:05.738 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=1 2024-09-14T08:56:05.738 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=1 2024-09-14T08:56:05.738 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T08:56:05.754 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:56:05.754 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:56:05.755 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:56:05.755 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_13 2024-09-14T08:56:05.838 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_13'] 2024-09-14T08:56:05.838 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:56:05.838 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:56:05.842 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:56:05.842 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_13 2024-09-14T08:56:05.842 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:2 2024-09-14T08:56:05.842 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:56:05.842 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:56:05.842 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-14T08:56:05.842 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:2 2024-09-14T08:56:05.842 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:56:05.842 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:56:05.842 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:56:05.843 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=2 2024-09-14T08:56:05.843 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T08:56:05.843 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:56:05.843 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:56:05.843 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=2 2024-09-14T08:56:05.843 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:56:05.843 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-14T08:56:05.843 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:2 2024-09-14T08:56:05.843 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:56:05.843 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:56:05.843 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T08:56:05.843 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:2 2024-09-14T08:56:05.843 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:56:05.843 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:56:05.843 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:56:05.843 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:56:05.843 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-14T08:56:05.844 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:56:05.844 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:56:05.844 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=2 2024-09-14T08:56:05.844 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=2 2024-09-14T08:56:05.844 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T08:56:05.861 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:56:05.861 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:56:05.861 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:56:05.861 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_13 2024-09-14T08:56:05.949 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ return 0 2024-09-14T08:56:05.949 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T08:56:05.949 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ echo ' "state": "Replaying"' 2024-09-14T08:56:05.950 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep -i Replaying 2024-09-14T08:56:05.950 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: "state": "Replaying" 2024-09-14T08:56:05.952 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_state=started 2024-09-14T08:56:05.952 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test started = started 2024-09-14T08:56:05.952 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:56:05.952 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_13 2024-09-14T08:56:05.952 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:56:05.952 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:56:05.952 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:05.952 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_13 2024-09-14T08:56:05.952 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = journal ']' 2024-09-14T08:56:05.952 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = snapshot ']' 2024-09-14T08:56:05.952 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_13 2024-09-14T08:56:05.953 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:56:05.953 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:56:05.953 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:05.953 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_13 2024-09-14T08:56:05.953 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster2-mirror-image_13.status 2024-09-14T08:56:05.953 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster2-mirror-image_13.status 2024-09-14T08:56:05.953 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:56:05.954 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_13.status 2024-09-14T08:56:05.954 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster1-mirror-image_13.status 2024-09-14T08:56:05.955 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster1-mirror-image_13.status 2024-09-14T08:56:05.955 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:56:05.955 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_status_log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_13.status 2024-09-14T08:56:05.955 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ mirror_image_snapshot cluster2 mirror image_13 2024-09-14T08:56:05.955 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:56:05.955 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:05.955 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_13 2024-09-14T08:56:05.955 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_13 2024-09-14T08:56:06.158 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.0 2024-09-14T08:56:06.159 INFO:teuthology.orchestra.run:waiting for 600 2024-09-14T08:56:06.159 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.0 2024-09-14T08:56:06.159 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Restarting daemon 2024-09-14T08:56:06.159 DEBUG:teuthology.orchestra.run.smithi164:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 2024-09-14T08:56:06.162 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Started 2024-09-14T08:56:06.162 INFO:tasks.rbd_mirror_thrash:waiting for 74 secs before thrashing 2024-09-14T08:56:06.642 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Snapshot ID: 110 2024-09-14T08:56:06.648 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_13 /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_13.status 2024-09-14T08:56:06.649 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:56:06.649 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:06.649 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_13 2024-09-14T08:56:06.649 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_13.status 2024-09-14T08:56:06.649 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_13 --format xml 2024-09-14T08:56:06.649 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:56:06.695 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-14T08:56:06.696 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snapshot_id=110 2024-09-14T08:56:06.697 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:56:06.697 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:56:06.697 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.2 2024-09-14T08:56:06.898 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_13 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_13.status 2024-09-14T08:56:06.898 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:56:06.898 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:06.898 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_13 2024-09-14T08:56:06.898 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_13.status 2024-09-14T08:56:06.898 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_13 --format xml 2024-09-14T08:56:06.898 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:56:06.947 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:56:06.948 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=47 2024-09-14T08:56:06.948 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 110 = 47 2024-09-14T08:56:06.949 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:56:06.949 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.4 2024-09-14T08:56:07.350 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_13 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_13.status 2024-09-14T08:56:07.350 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:56:07.350 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:07.350 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_13 2024-09-14T08:56:07.350 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_13.status 2024-09-14T08:56:07.350 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_13 --format xml 2024-09-14T08:56:07.350 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:56:07.400 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:56:07.402 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=47 2024-09-14T08:56:07.402 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 110 = 47 2024-09-14T08:56:07.402 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:56:07.402 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.8 2024-09-14T08:56:08.204 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_13 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_13.status 2024-09-14T08:56:08.204 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:56:08.204 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:08.204 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_13 2024-09-14T08:56:08.204 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_13.status 2024-09-14T08:56:08.204 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_13 --format xml 2024-09-14T08:56:08.204 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:56:08.257 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:56:08.257 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:56:08.258 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:56:08.258 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:56:08.258 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:56:08.259 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:56:08.259 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 110 = '' 2024-09-14T08:56:08.259 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:56:08.259 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1.6 2024-09-14T08:56:09.860 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_13 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_13.status 2024-09-14T08:56:09.860 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:56:09.860 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:09.860 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_13 2024-09-14T08:56:09.860 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_13.status 2024-09-14T08:56:09.861 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_13 --format xml 2024-09-14T08:56:09.861 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:56:09.916 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:56:09.918 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=110 2024-09-14T08:56:09.918 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 110 = 110 2024-09-14T08:56:09.918 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:56:09.918 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snap_present cluster1 mirror image_13 snap 2024-09-14T08:56:09.919 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:56:09.919 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:09.919 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_13 2024-09-14T08:56:09.919 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:56:09.919 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:56:09.919 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T08:56:09.919 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:56:10.919 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 info mirror/image_13@snap 2024-09-14T08:56:10.967 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_13': 2024-09-14T08:56:10.967 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T08:56:10.967 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:56:10.967 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 2 2024-09-14T08:56:10.967 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 10bd9cb98fee 2024-09-14T08:56:10.967 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.10bd9cb98fee 2024-09-14T08:56:10.967 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:56:10.968 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering, non-primary 2024-09-14T08:56:10.968 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:56:10.968 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:56:10.968 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:47:02 2024 2024-09-14T08:56:10.968 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:47:02 2024 2024-09-14T08:56:10.968 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:47:02 2024 2024-09-14T08:56:10.968 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: protected: False 2024-09-14T08:56:10.968 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:56:10.968 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:56:10.968 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: 5dc0c52c-6377-4410-90b1-49344e6d9458 2024-09-14T08:56:10.968 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: false 2024-09-14T08:56:10.971 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:56:10.971 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ compare_image_snaps mirror image_13 snap 2024-09-14T08:56:10.971 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:10.971 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_13 2024-09-14T08:56:10.971 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:56:10.972 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local ret=0 2024-09-14T08:56:10.972 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local rmt_export=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_13.export 2024-09-14T08:56:10.972 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local loc_export=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_13.export 2024-09-14T08:56:10.972 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_13.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_13.export 2024-09-14T08:56:10.972 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror export image_13@snap /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_13.export 2024-09-14T08:56:11.167 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T08:56:11.173 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 -p mirror export image_13@snap /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_13.export 2024-09-14T08:56:11.420 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T08:56:11.427 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cmp /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_13.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_13.export 2024-09-14T08:56:11.483 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_13.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_13.export 2024-09-14T08:56:11.521 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:56:11.521 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T08:56:11.521 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_14 2024-09-14T08:56:11.521 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_snap cluster2 mirror image_14 snap 2024-09-14T08:56:11.521 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:56:11.522 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:11.522 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_14 2024-09-14T08:56:11.522 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:56:11.522 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror snap create image_14@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T08:56:11.656 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_started cluster1 mirror image_14 2024-09-14T08:56:11.656 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:56:11.656 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:11.656 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_14 2024-09-14T08:56:11.656 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_state cluster1 mirror image_14 started 2024-09-14T08:56:11.656 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:56:11.656 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:11.656 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_14 2024-09-14T08:56:11.656 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=started 2024-09-14T08:56:11.656 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:56:11.656 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:56:11.656 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:56:12.657 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_replay_state cluster1 mirror image_14 started 2024-09-14T08:56:12.657 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:56:12.657 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:12.657 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_14 2024-09-14T08:56:12.657 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=started 2024-09-14T08:56:12.657 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_result 2024-09-14T08:56:12.657 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=stopped 2024-09-14T08:56:12.658 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_14 2024-09-14T08:56:12.658 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_instance=cluster1 2024-09-14T08:56:12.659 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:56:12.659 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ grep -i state 2024-09-14T08:56:12.659 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T08:56:12.659 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance=cluster1 2024-09-14T08:56:12.659 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local loop_instance 2024-09-14T08:56:12.659 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:56:12.659 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sleep 0 2024-09-14T08:56:12.659 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T08:56:12.659 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ seq 0 3 2024-09-14T08:56:12.662 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:56:12.662 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_14 2024-09-14T08:56:12.662 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:0 2024-09-14T08:56:12.662 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:56:12.662 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:56:12.662 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T08:56:12.662 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:0 2024-09-14T08:56:12.662 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:56:12.663 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:56:12.663 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:56:12.663 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=0 2024-09-14T08:56:12.663 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T08:56:12.663 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:56:12.663 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:56:12.663 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=0 2024-09-14T08:56:12.663 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:56:12.663 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T08:56:12.663 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:0 2024-09-14T08:56:12.663 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:56:12.663 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:56:12.663 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T08:56:12.663 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:0 2024-09-14T08:56:12.663 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:56:12.664 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:56:12.664 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:56:12.664 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:56:12.664 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T08:56:12.664 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:56:12.664 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:56:12.664 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=0 2024-09-14T08:56:12.664 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=0 2024-09-14T08:56:12.664 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T08:56:12.684 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:56:12.684 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:56:12.684 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:56:12.684 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_14 2024-09-14T08:56:12.764 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_14'] 2024-09-14T08:56:12.764 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:56:12.764 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:56:12.767 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:56:12.767 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_14 2024-09-14T08:56:12.767 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:1 2024-09-14T08:56:12.767 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:56:12.767 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:56:12.767 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T08:56:12.768 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:1 2024-09-14T08:56:12.768 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:56:12.768 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:56:12.768 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:56:12.768 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=1 2024-09-14T08:56:12.768 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T08:56:12.768 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:56:12.768 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:56:12.768 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=1 2024-09-14T08:56:12.768 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:56:12.768 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T08:56:12.768 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:1 2024-09-14T08:56:12.768 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:56:12.768 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:56:12.769 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T08:56:12.769 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:1 2024-09-14T08:56:12.769 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:56:12.769 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:56:12.769 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:56:12.769 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:56:12.769 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T08:56:12.769 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:56:12.769 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:56:12.769 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=1 2024-09-14T08:56:12.769 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=1 2024-09-14T08:56:12.769 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T08:56:12.786 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:56:12.786 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:56:12.786 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:56:12.786 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_14 2024-09-14T08:56:12.867 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_14'] 2024-09-14T08:56:12.868 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:56:12.868 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:56:12.871 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:56:12.871 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_14 2024-09-14T08:56:12.871 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:2 2024-09-14T08:56:12.871 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:56:12.871 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:56:12.871 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-14T08:56:12.871 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:2 2024-09-14T08:56:12.871 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:56:12.871 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:56:12.871 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:56:12.871 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=2 2024-09-14T08:56:12.872 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T08:56:12.872 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:56:12.872 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:56:12.872 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=2 2024-09-14T08:56:12.872 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:56:12.872 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-14T08:56:12.872 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:2 2024-09-14T08:56:12.872 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:56:12.872 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:56:12.872 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T08:56:12.872 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:2 2024-09-14T08:56:12.872 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:56:12.872 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:56:12.872 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:56:12.873 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:56:12.873 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-14T08:56:12.873 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:56:12.873 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:56:12.873 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=2 2024-09-14T08:56:12.873 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=2 2024-09-14T08:56:12.873 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T08:56:12.889 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:56:12.889 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:56:12.889 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:56:12.889 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_14 2024-09-14T08:56:12.970 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_14'] 2024-09-14T08:56:12.970 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:56:12.970 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:56:12.973 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:56:12.973 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_14 2024-09-14T08:56:12.973 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:3 2024-09-14T08:56:12.973 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:56:12.973 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:56:12.973 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:3 cluster instance 2024-09-14T08:56:12.973 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:3 2024-09-14T08:56:12.973 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:56:12.973 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:56:12.974 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:56:12.974 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=3 2024-09-14T08:56:12.974 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-14T08:56:12.974 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:56:12.974 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:56:12.974 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=3 2024-09-14T08:56:12.974 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=3 2024-09-14T08:56:12.974 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:3 cluster1 2024-09-14T08:56:12.974 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:3 2024-09-14T08:56:12.974 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:56:12.974 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:56:12.974 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-14T08:56:12.974 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:3 2024-09-14T08:56:12.974 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:56:12.975 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:56:12.975 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:56:12.975 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=3 2024-09-14T08:56:12.975 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 3 = cluster1:3 ']' 2024-09-14T08:56:12.975 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:56:12.975 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:56:12.975 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=3 2024-09-14T08:56:12.975 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=3 2024-09-14T08:56:12.975 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-14T08:56:12.992 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T08:56:12.992 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T08:56:12.992 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T08:56:12.992 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_14 2024-09-14T08:56:13.081 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ return 0 2024-09-14T08:56:13.082 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T08:56:13.082 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ echo ' "state": "Replaying"' 2024-09-14T08:56:13.082 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep -i Replaying 2024-09-14T08:56:13.083 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: "state": "Replaying" 2024-09-14T08:56:13.084 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_state=started 2024-09-14T08:56:13.084 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test started = started 2024-09-14T08:56:13.084 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:56:13.084 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_14 2024-09-14T08:56:13.084 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:56:13.084 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:56:13.085 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:13.085 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_14 2024-09-14T08:56:13.085 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = journal ']' 2024-09-14T08:56:13.085 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = snapshot ']' 2024-09-14T08:56:13.085 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_14 2024-09-14T08:56:13.085 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:56:13.085 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:56:13.085 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:13.085 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_14 2024-09-14T08:56:13.085 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster2-mirror-image_14.status 2024-09-14T08:56:13.085 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster2-mirror-image_14.status 2024-09-14T08:56:13.085 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:56:13.087 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_14.status 2024-09-14T08:56:13.087 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster1-mirror-image_14.status 2024-09-14T08:56:13.087 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster1-mirror-image_14.status 2024-09-14T08:56:13.087 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:56:13.087 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_status_log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_14.status 2024-09-14T08:56:13.087 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ mirror_image_snapshot cluster2 mirror image_14 2024-09-14T08:56:13.087 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:56:13.087 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:13.087 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_14 2024-09-14T08:56:13.087 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_14 2024-09-14T08:56:13.664 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Snapshot ID: 112 2024-09-14T08:56:13.671 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_14 /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_14.status 2024-09-14T08:56:13.671 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:56:13.671 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:13.671 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_14 2024-09-14T08:56:13.671 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_14.status 2024-09-14T08:56:13.671 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_14 --format xml 2024-09-14T08:56:13.671 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:56:13.717 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-14T08:56:13.719 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snapshot_id=112 2024-09-14T08:56:13.719 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:56:13.719 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:56:13.719 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.2 2024-09-14T08:56:13.920 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_14 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_14.status 2024-09-14T08:56:13.920 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:56:13.921 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:13.921 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_14 2024-09-14T08:56:13.921 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_14.status 2024-09-14T08:56:13.921 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_14 --format xml 2024-09-14T08:56:13.921 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:56:13.971 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:56:13.972 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=48 2024-09-14T08:56:13.973 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 112 = 48 2024-09-14T08:56:13.973 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:56:13.973 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.4 2024-09-14T08:56:14.374 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_14 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_14.status 2024-09-14T08:56:14.374 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:56:14.374 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:14.374 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_14 2024-09-14T08:56:14.374 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_14.status 2024-09-14T08:56:14.374 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_14 --format xml 2024-09-14T08:56:14.374 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:56:14.424 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:56:14.426 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=48 2024-09-14T08:56:14.426 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 112 = 48 2024-09-14T08:56:14.426 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:56:14.426 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.8 2024-09-14T08:56:15.227 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_14 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_14.status 2024-09-14T08:56:15.227 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:56:15.227 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:15.227 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_14 2024-09-14T08:56:15.227 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_14.status 2024-09-14T08:56:15.228 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_14 --format xml 2024-09-14T08:56:15.228 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:56:15.283 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:56:15.283 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:56:15.285 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:56:15.285 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:56:15.285 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:56:15.286 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:56:15.286 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 112 = '' 2024-09-14T08:56:15.286 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:56:15.286 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1.6 2024-09-14T08:56:16.887 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_14 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_14.status 2024-09-14T08:56:16.887 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:56:16.887 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:16.888 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_14 2024-09-14T08:56:16.888 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_14.status 2024-09-14T08:56:16.888 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_14 --format xml 2024-09-14T08:56:16.888 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:56:16.942 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:56:16.945 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=112 2024-09-14T08:56:16.945 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 112 = 112 2024-09-14T08:56:16.945 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:56:16.945 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snap_present cluster1 mirror image_14 snap 2024-09-14T08:56:16.945 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:56:16.945 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:16.945 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_14 2024-09-14T08:56:16.945 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:56:16.945 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:56:16.945 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T08:56:16.946 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:56:17.945 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 info mirror/image_14@snap 2024-09-14T08:56:17.995 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_14': 2024-09-14T08:56:17.995 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T08:56:17.995 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:56:17.995 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 2 2024-09-14T08:56:17.995 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 1287b6a5b79 2024-09-14T08:56:17.995 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.1287b6a5b79 2024-09-14T08:56:17.995 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:56:17.995 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering, non-primary 2024-09-14T08:56:17.995 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:56:17.995 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:56:17.995 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:47:14 2024 2024-09-14T08:56:17.996 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:47:14 2024 2024-09-14T08:56:17.996 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:47:14 2024 2024-09-14T08:56:17.996 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: protected: False 2024-09-14T08:56:17.996 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:56:17.996 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:56:17.996 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: d055c61d-091e-4c03-a847-749c1be38f92 2024-09-14T08:56:17.996 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: false 2024-09-14T08:56:17.999 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:56:17.999 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ compare_image_snaps mirror image_14 snap 2024-09-14T08:56:17.999 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:17.999 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_14 2024-09-14T08:56:17.999 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:56:18.000 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local ret=0 2024-09-14T08:56:18.000 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local rmt_export=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_14.export 2024-09-14T08:56:18.000 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local loc_export=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_14.export 2024-09-14T08:56:18.000 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_14.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_14.export 2024-09-14T08:56:18.000 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror export image_14@snap /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_14.export 2024-09-14T08:56:18.191 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T08:56:18.198 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 -p mirror export image_14@snap /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_14.export 2024-09-14T08:56:18.439 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T08:56:18.445 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cmp /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_14.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_14.export 2024-09-14T08:56:18.500 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_14.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_14.export 2024-09-14T08:56:18.532 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:56:18.532 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T08:56:18.532 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_15 2024-09-14T08:56:18.533 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_snap cluster2 mirror image_15 snap 2024-09-14T08:56:18.533 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:56:18.533 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:18.533 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_15 2024-09-14T08:56:18.533 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:56:18.533 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror snap create image_15@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T08:56:19.218 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_started cluster1 mirror image_15 2024-09-14T08:56:19.218 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:56:19.218 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:19.218 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_15 2024-09-14T08:56:19.219 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_state cluster1 mirror image_15 started 2024-09-14T08:56:19.219 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:56:19.219 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:19.219 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_15 2024-09-14T08:56:19.219 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=started 2024-09-14T08:56:19.219 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:56:19.219 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:56:19.219 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:56:20.220 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_replay_state cluster1 mirror image_15 started 2024-09-14T08:56:20.220 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:56:20.220 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:20.221 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_15 2024-09-14T08:56:20.221 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=started 2024-09-14T08:56:20.221 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_result 2024-09-14T08:56:20.221 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=stopped 2024-09-14T08:56:20.221 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_15 2024-09-14T08:56:20.221 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_instance=cluster1 2024-09-14T08:56:20.221 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:56:20.221 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ grep -i state 2024-09-14T08:56:20.221 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T08:56:20.221 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance=cluster1 2024-09-14T08:56:20.221 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local loop_instance 2024-09-14T08:56:20.221 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:56:20.221 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sleep 0 2024-09-14T08:56:20.224 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T08:56:20.224 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ seq 0 3 2024-09-14T08:56:20.225 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:56:20.225 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_15 2024-09-14T08:56:20.225 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:0 2024-09-14T08:56:20.225 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:56:20.225 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:56:20.225 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T08:56:20.225 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:0 2024-09-14T08:56:20.225 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:56:20.225 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:56:20.225 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:56:20.225 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=0 2024-09-14T08:56:20.225 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T08:56:20.226 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:56:20.226 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:56:20.226 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=0 2024-09-14T08:56:20.226 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:56:20.226 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T08:56:20.226 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:0 2024-09-14T08:56:20.226 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:56:20.226 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:56:20.226 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T08:56:20.226 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:0 2024-09-14T08:56:20.226 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:56:20.226 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:56:20.226 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:56:20.226 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:56:20.227 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T08:56:20.227 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:56:20.227 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:56:20.227 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=0 2024-09-14T08:56:20.227 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=0 2024-09-14T08:56:20.227 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T08:56:20.246 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:56:20.246 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:56:20.246 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:56:20.247 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_15 2024-09-14T08:56:20.331 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_15'] 2024-09-14T08:56:20.331 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:56:20.332 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:56:20.334 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:56:20.334 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_15 2024-09-14T08:56:20.335 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:1 2024-09-14T08:56:20.335 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:56:20.335 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:56:20.335 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T08:56:20.335 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:1 2024-09-14T08:56:20.335 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:56:20.335 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:56:20.335 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:56:20.335 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=1 2024-09-14T08:56:20.335 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T08:56:20.335 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:56:20.335 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:56:20.335 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=1 2024-09-14T08:56:20.335 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:56:20.336 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T08:56:20.336 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:1 2024-09-14T08:56:20.336 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:56:20.336 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:56:20.336 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T08:56:20.336 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:1 2024-09-14T08:56:20.336 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:56:20.336 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:56:20.336 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:56:20.336 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:56:20.336 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T08:56:20.336 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:56:20.336 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:56:20.336 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=1 2024-09-14T08:56:20.337 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=1 2024-09-14T08:56:20.337 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T08:56:20.354 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:56:20.354 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:56:20.354 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:56:20.354 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_15 2024-09-14T08:56:20.434 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_15'] 2024-09-14T08:56:20.434 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:56:20.434 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:56:20.437 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:56:20.437 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_15 2024-09-14T08:56:20.437 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:2 2024-09-14T08:56:20.437 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:56:20.437 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:56:20.438 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-14T08:56:20.438 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:2 2024-09-14T08:56:20.438 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:56:20.438 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:56:20.438 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:56:20.438 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=2 2024-09-14T08:56:20.438 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T08:56:20.438 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:56:20.438 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:56:20.438 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=2 2024-09-14T08:56:20.438 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:56:20.438 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-14T08:56:20.438 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:2 2024-09-14T08:56:20.438 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:56:20.439 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:56:20.439 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T08:56:20.439 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:2 2024-09-14T08:56:20.439 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:56:20.439 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:56:20.439 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:56:20.439 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:56:20.439 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-14T08:56:20.439 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:56:20.439 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:56:20.439 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=2 2024-09-14T08:56:20.439 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=2 2024-09-14T08:56:20.439 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T08:56:20.457 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:56:20.457 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:56:20.457 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:56:20.457 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_15 2024-09-14T08:56:20.543 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ return 0 2024-09-14T08:56:20.544 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T08:56:20.544 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ echo ' "state": "Replaying"' 2024-09-14T08:56:20.544 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep -i Replaying 2024-09-14T08:56:20.544 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: "state": "Replaying" 2024-09-14T08:56:20.546 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_state=started 2024-09-14T08:56:20.546 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test started = started 2024-09-14T08:56:20.546 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:56:20.546 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_15 2024-09-14T08:56:20.546 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:56:20.546 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:56:20.546 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:20.546 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_15 2024-09-14T08:56:20.546 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = journal ']' 2024-09-14T08:56:20.547 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = snapshot ']' 2024-09-14T08:56:20.547 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_15 2024-09-14T08:56:20.547 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:56:20.547 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:56:20.547 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:20.547 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_15 2024-09-14T08:56:20.547 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster2-mirror-image_15.status 2024-09-14T08:56:20.547 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster2-mirror-image_15.status 2024-09-14T08:56:20.547 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:56:20.548 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_15.status 2024-09-14T08:56:20.548 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster1-mirror-image_15.status 2024-09-14T08:56:20.549 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster1-mirror-image_15.status 2024-09-14T08:56:20.549 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:56:20.549 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_status_log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_15.status 2024-09-14T08:56:20.549 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ mirror_image_snapshot cluster2 mirror image_15 2024-09-14T08:56:20.549 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:56:20.549 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:20.549 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_15 2024-09-14T08:56:20.549 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_15 2024-09-14T08:56:21.226 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Snapshot ID: 114 2024-09-14T08:56:21.233 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_15 /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_15.status 2024-09-14T08:56:21.233 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:56:21.233 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:21.233 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_15 2024-09-14T08:56:21.233 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_15.status 2024-09-14T08:56:21.233 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_15 --format xml 2024-09-14T08:56:21.234 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:56:21.283 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-14T08:56:21.284 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snapshot_id=114 2024-09-14T08:56:21.284 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:56:21.284 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:56:21.285 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.2 2024-09-14T08:56:21.486 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_15 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_15.status 2024-09-14T08:56:21.486 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:56:21.486 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:21.486 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_15 2024-09-14T08:56:21.486 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_15.status 2024-09-14T08:56:21.487 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_15 --format xml 2024-09-14T08:56:21.487 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:56:21.541 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:56:21.542 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:56:21.543 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:56:21.543 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:56:21.543 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:56:21.544 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:56:21.544 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 114 = '' 2024-09-14T08:56:21.544 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:56:21.544 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.4 2024-09-14T08:56:21.945 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_15 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_15.status 2024-09-14T08:56:21.945 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:56:21.945 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:21.945 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_15 2024-09-14T08:56:21.945 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_15.status 2024-09-14T08:56:21.945 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_15 --format xml 2024-09-14T08:56:21.945 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:56:21.996 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:56:21.996 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:56:21.997 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:56:21.997 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:56:21.997 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:56:21.998 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:56:21.998 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 114 = '' 2024-09-14T08:56:21.998 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:56:21.998 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.8 2024-09-14T08:56:22.799 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_15 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_15.status 2024-09-14T08:56:22.799 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:56:22.799 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:22.799 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_15 2024-09-14T08:56:22.799 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_15.status 2024-09-14T08:56:22.800 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_15 --format xml 2024-09-14T08:56:22.800 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:56:22.846 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:56:22.848 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=114 2024-09-14T08:56:22.848 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 114 = 114 2024-09-14T08:56:22.849 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:56:22.849 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snap_present cluster1 mirror image_15 snap 2024-09-14T08:56:22.849 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:56:22.849 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:22.849 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_15 2024-09-14T08:56:22.849 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:56:22.849 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:56:22.849 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T08:56:22.849 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:56:23.849 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 info mirror/image_15@snap 2024-09-14T08:56:23.899 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_15': 2024-09-14T08:56:23.899 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T08:56:23.900 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:56:23.900 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 2 2024-09-14T08:56:23.900 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 1287759fe7c3 2024-09-14T08:56:23.900 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.1287759fe7c3 2024-09-14T08:56:23.900 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:56:23.900 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering, non-primary 2024-09-14T08:56:23.900 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:56:23.900 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:56:23.900 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:47:28 2024 2024-09-14T08:56:23.900 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:47:28 2024 2024-09-14T08:56:23.900 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:47:28 2024 2024-09-14T08:56:23.900 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: protected: False 2024-09-14T08:56:23.901 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:56:23.901 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:56:23.901 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: b5a7a156-c2fc-47f1-b818-af6739582d53 2024-09-14T08:56:23.901 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: false 2024-09-14T08:56:23.903 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:56:23.904 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ compare_image_snaps mirror image_15 snap 2024-09-14T08:56:23.904 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:23.904 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_15 2024-09-14T08:56:23.904 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:56:23.904 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local ret=0 2024-09-14T08:56:23.904 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local rmt_export=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_15.export 2024-09-14T08:56:23.905 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local loc_export=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_15.export 2024-09-14T08:56:23.905 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_15.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_15.export 2024-09-14T08:56:23.905 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror export image_15@snap /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_15.export 2024-09-14T08:56:24.100 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T08:56:24.107 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 -p mirror export image_15@snap /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_15.export 2024-09-14T08:56:24.360 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T08:56:24.367 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cmp /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_15.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_15.export 2024-09-14T08:56:24.421 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_15.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_15.export 2024-09-14T08:56:24.453 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:56:24.453 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T08:56:24.453 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_16 2024-09-14T08:56:24.453 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_snap cluster2 mirror image_16 snap 2024-09-14T08:56:24.453 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:56:24.454 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:24.454 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_16 2024-09-14T08:56:24.454 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:56:24.454 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror snap create image_16@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T08:56:25.229 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_started cluster1 mirror image_16 2024-09-14T08:56:25.230 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:56:25.230 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:25.230 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_16 2024-09-14T08:56:25.230 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_state cluster1 mirror image_16 started 2024-09-14T08:56:25.230 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:56:25.230 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:25.230 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_16 2024-09-14T08:56:25.230 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=started 2024-09-14T08:56:25.230 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:56:25.230 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:56:25.230 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:56:26.230 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_replay_state cluster1 mirror image_16 started 2024-09-14T08:56:26.230 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:56:26.230 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:26.230 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_16 2024-09-14T08:56:26.230 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=started 2024-09-14T08:56:26.230 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_result 2024-09-14T08:56:26.230 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=stopped 2024-09-14T08:56:26.230 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_16 2024-09-14T08:56:26.230 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ grep -i state 2024-09-14T08:56:26.230 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_instance=cluster1 2024-09-14T08:56:26.230 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:56:26.230 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T08:56:26.231 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance=cluster1 2024-09-14T08:56:26.231 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local loop_instance 2024-09-14T08:56:26.231 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:56:26.231 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sleep 0 2024-09-14T08:56:26.232 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T08:56:26.232 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ seq 0 3 2024-09-14T08:56:26.232 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:56:26.233 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_16 2024-09-14T08:56:26.233 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:0 2024-09-14T08:56:26.233 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:56:26.233 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:56:26.233 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T08:56:26.233 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:0 2024-09-14T08:56:26.233 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:56:26.233 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:56:26.233 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:56:26.233 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=0 2024-09-14T08:56:26.233 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T08:56:26.233 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:56:26.233 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:56:26.234 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=0 2024-09-14T08:56:26.234 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:56:26.235 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T08:56:26.235 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:0 2024-09-14T08:56:26.236 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:56:26.236 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:56:26.236 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T08:56:26.236 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:0 2024-09-14T08:56:26.236 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:56:26.236 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:56:26.236 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:56:26.236 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:56:26.236 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T08:56:26.236 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:56:26.236 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:56:26.236 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=0 2024-09-14T08:56:26.236 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=0 2024-09-14T08:56:26.236 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T08:56:26.254 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:56:26.254 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:56:26.254 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:56:26.254 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_16 2024-09-14T08:56:26.339 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_16'] 2024-09-14T08:56:26.339 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:56:26.339 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:56:26.342 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:56:26.342 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_16 2024-09-14T08:56:26.342 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:1 2024-09-14T08:56:26.343 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:56:26.343 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:56:26.343 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T08:56:26.343 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:1 2024-09-14T08:56:26.343 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:56:26.343 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:56:26.343 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:56:26.343 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=1 2024-09-14T08:56:26.343 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T08:56:26.343 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:56:26.343 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:56:26.343 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=1 2024-09-14T08:56:26.343 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:56:26.343 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T08:56:26.344 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:1 2024-09-14T08:56:26.344 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:56:26.344 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:56:26.344 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T08:56:26.344 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:1 2024-09-14T08:56:26.344 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:56:26.344 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:56:26.344 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:56:26.344 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:56:26.344 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T08:56:26.344 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:56:26.344 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:56:26.344 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=1 2024-09-14T08:56:26.345 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=1 2024-09-14T08:56:26.345 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T08:56:26.361 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:56:26.361 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:56:26.361 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:56:26.361 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_16 2024-09-14T08:56:26.443 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_16'] 2024-09-14T08:56:26.443 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:56:26.443 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:56:26.446 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:56:26.446 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_16 2024-09-14T08:56:26.446 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:2 2024-09-14T08:56:26.446 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:56:26.447 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:56:26.447 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-14T08:56:26.447 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:2 2024-09-14T08:56:26.447 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:56:26.447 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:56:26.447 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:56:26.447 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=2 2024-09-14T08:56:26.447 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T08:56:26.447 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:56:26.447 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:56:26.447 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=2 2024-09-14T08:56:26.447 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:56:26.447 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-14T08:56:26.447 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:2 2024-09-14T08:56:26.448 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:56:26.448 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:56:26.448 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T08:56:26.448 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:2 2024-09-14T08:56:26.448 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:56:26.448 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:56:26.448 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:56:26.448 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:56:26.448 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-14T08:56:26.448 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:56:26.448 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:56:26.448 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=2 2024-09-14T08:56:26.449 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=2 2024-09-14T08:56:26.449 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T08:56:26.464 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:56:26.464 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:56:26.465 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:56:26.465 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_16 2024-09-14T08:56:26.544 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_16'] 2024-09-14T08:56:26.544 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:56:26.544 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:56:26.547 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:56:26.547 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_16 2024-09-14T08:56:26.547 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:3 2024-09-14T08:56:26.547 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:56:26.548 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:56:26.548 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:3 cluster instance 2024-09-14T08:56:26.548 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:3 2024-09-14T08:56:26.548 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:56:26.548 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:56:26.548 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:56:26.548 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=3 2024-09-14T08:56:26.548 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-14T08:56:26.548 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:56:26.548 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:56:26.548 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=3 2024-09-14T08:56:26.549 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=3 2024-09-14T08:56:26.549 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:3 cluster1 2024-09-14T08:56:26.549 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:3 2024-09-14T08:56:26.549 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:56:26.549 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:56:26.549 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-14T08:56:26.549 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:3 2024-09-14T08:56:26.549 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:56:26.549 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:56:26.549 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:56:26.549 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=3 2024-09-14T08:56:26.549 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 3 = cluster1:3 ']' 2024-09-14T08:56:26.550 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:56:26.550 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:56:26.550 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=3 2024-09-14T08:56:26.550 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=3 2024-09-14T08:56:26.550 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-14T08:56:26.565 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T08:56:26.565 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T08:56:26.565 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T08:56:26.566 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_16 2024-09-14T08:56:26.645 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_16'] 2024-09-14T08:56:26.645 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:56:26.645 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:56:26.646 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:56:26.646 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sleep 1 2024-09-14T08:56:27.648 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T08:56:27.648 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ seq 0 3 2024-09-14T08:56:27.651 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:56:27.651 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_16 2024-09-14T08:56:27.651 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:0 2024-09-14T08:56:27.651 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:56:27.651 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:56:27.651 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T08:56:27.651 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:0 2024-09-14T08:56:27.651 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:56:27.651 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:56:27.651 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:56:27.651 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=0 2024-09-14T08:56:27.652 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T08:56:27.652 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:56:27.652 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:56:27.652 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=0 2024-09-14T08:56:27.652 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:56:27.652 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T08:56:27.652 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:0 2024-09-14T08:56:27.652 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:56:27.652 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:56:27.652 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T08:56:27.652 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:0 2024-09-14T08:56:27.652 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:56:27.652 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:56:27.652 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:56:27.653 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:56:27.653 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T08:56:27.653 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:56:27.653 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:56:27.653 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=0 2024-09-14T08:56:27.653 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=0 2024-09-14T08:56:27.653 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T08:56:27.670 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:56:27.670 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:56:27.670 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:56:27.670 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_16 2024-09-14T08:56:27.750 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_16'] 2024-09-14T08:56:27.751 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:56:27.751 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:56:27.753 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:56:27.754 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_16 2024-09-14T08:56:27.754 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:1 2024-09-14T08:56:27.754 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:56:27.754 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:56:27.754 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T08:56:27.754 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:1 2024-09-14T08:56:27.754 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:56:27.754 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:56:27.754 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:56:27.754 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=1 2024-09-14T08:56:27.754 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T08:56:27.754 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:56:27.755 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:56:27.755 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=1 2024-09-14T08:56:27.755 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:56:27.755 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T08:56:27.755 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:1 2024-09-14T08:56:27.755 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:56:27.755 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:56:27.755 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T08:56:27.755 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:1 2024-09-14T08:56:27.755 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:56:27.755 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:56:27.755 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:56:27.755 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:56:27.755 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T08:56:27.755 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:56:27.756 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:56:27.756 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=1 2024-09-14T08:56:27.756 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=1 2024-09-14T08:56:27.756 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T08:56:27.771 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:56:27.772 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:56:27.772 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:56:27.772 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_16 2024-09-14T08:56:27.851 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_16'] 2024-09-14T08:56:27.851 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:56:27.851 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:56:27.854 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:56:27.854 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_16 2024-09-14T08:56:27.854 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:2 2024-09-14T08:56:27.854 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:56:27.854 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:56:27.854 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-14T08:56:27.854 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:2 2024-09-14T08:56:27.854 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:56:27.854 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:56:27.854 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:56:27.855 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=2 2024-09-14T08:56:27.855 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T08:56:27.855 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:56:27.855 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:56:27.855 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=2 2024-09-14T08:56:27.855 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:56:27.855 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-14T08:56:27.855 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:2 2024-09-14T08:56:27.855 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:56:27.855 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:56:27.855 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T08:56:27.855 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:2 2024-09-14T08:56:27.855 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:56:27.855 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:56:27.856 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:56:27.856 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:56:27.856 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-14T08:56:27.856 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:56:27.856 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:56:27.856 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=2 2024-09-14T08:56:27.856 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=2 2024-09-14T08:56:27.856 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T08:56:27.873 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:56:27.873 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:56:27.873 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:56:27.873 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_16 2024-09-14T08:56:27.951 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_16'] 2024-09-14T08:56:27.951 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:56:27.951 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:56:27.955 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:56:27.955 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_16 2024-09-14T08:56:27.955 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:3 2024-09-14T08:56:27.955 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:56:27.955 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:56:27.955 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:3 cluster instance 2024-09-14T08:56:27.955 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:3 2024-09-14T08:56:27.955 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:56:27.955 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:56:27.955 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:56:27.955 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=3 2024-09-14T08:56:27.955 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-14T08:56:27.956 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:56:27.956 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:56:27.956 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=3 2024-09-14T08:56:27.956 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=3 2024-09-14T08:56:27.956 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:3 cluster1 2024-09-14T08:56:27.956 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:3 2024-09-14T08:56:27.956 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:56:27.956 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:56:27.956 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-14T08:56:27.956 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:3 2024-09-14T08:56:27.956 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:56:27.956 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:56:27.956 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:56:27.956 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=3 2024-09-14T08:56:27.957 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 3 = cluster1:3 ']' 2024-09-14T08:56:27.957 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:56:27.957 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:56:27.957 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=3 2024-09-14T08:56:27.957 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=3 2024-09-14T08:56:27.957 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-14T08:56:27.973 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T08:56:27.973 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T08:56:27.973 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T08:56:27.973 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_16 2024-09-14T08:56:28.054 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_16'] 2024-09-14T08:56:28.054 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:56:28.054 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:56:28.055 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:56:28.055 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sleep 2 2024-09-14T08:56:30.057 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T08:56:30.057 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ seq 0 3 2024-09-14T08:56:30.059 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:56:30.060 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_16 2024-09-14T08:56:30.060 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:0 2024-09-14T08:56:30.060 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:56:30.060 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:56:30.060 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T08:56:30.060 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:0 2024-09-14T08:56:30.060 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:56:30.060 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:56:30.060 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:56:30.060 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=0 2024-09-14T08:56:30.061 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T08:56:30.061 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:56:30.061 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:56:30.061 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=0 2024-09-14T08:56:30.061 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:56:30.061 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T08:56:30.061 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:0 2024-09-14T08:56:30.061 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:56:30.061 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:56:30.061 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T08:56:30.061 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:0 2024-09-14T08:56:30.061 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:56:30.062 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:56:30.062 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:56:30.062 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:56:30.062 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T08:56:30.062 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:56:30.062 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:56:30.062 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=0 2024-09-14T08:56:30.062 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=0 2024-09-14T08:56:30.062 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T08:56:30.082 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:56:30.082 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:56:30.082 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:56:30.082 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_16 2024-09-14T08:56:30.168 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_16'] 2024-09-14T08:56:30.168 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:56:30.168 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:56:30.171 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:56:30.171 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_16 2024-09-14T08:56:30.171 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:1 2024-09-14T08:56:30.171 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:56:30.171 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:56:30.171 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T08:56:30.171 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:1 2024-09-14T08:56:30.172 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:56:30.172 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:56:30.172 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:56:30.172 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=1 2024-09-14T08:56:30.172 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T08:56:30.172 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:56:30.172 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:56:30.172 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=1 2024-09-14T08:56:30.172 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:56:30.172 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T08:56:30.172 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:1 2024-09-14T08:56:30.172 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:56:30.172 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:56:30.173 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T08:56:30.173 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:1 2024-09-14T08:56:30.173 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:56:30.173 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:56:30.173 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:56:30.173 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:56:30.173 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T08:56:30.173 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:56:30.173 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:56:30.173 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=1 2024-09-14T08:56:30.173 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=1 2024-09-14T08:56:30.173 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T08:56:30.190 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:56:30.190 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:56:30.190 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:56:30.190 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_16 2024-09-14T08:56:30.270 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_16'] 2024-09-14T08:56:30.270 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:56:30.270 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:56:30.273 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:56:30.273 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_16 2024-09-14T08:56:30.273 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:2 2024-09-14T08:56:30.273 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:56:30.273 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:56:30.273 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-14T08:56:30.274 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:2 2024-09-14T08:56:30.274 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:56:30.274 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:56:30.274 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:56:30.274 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=2 2024-09-14T08:56:30.274 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T08:56:30.274 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:56:30.274 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:56:30.274 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=2 2024-09-14T08:56:30.274 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:56:30.274 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-14T08:56:30.274 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:2 2024-09-14T08:56:30.274 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:56:30.274 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:56:30.275 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T08:56:30.275 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:2 2024-09-14T08:56:30.275 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:56:30.275 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:56:30.275 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:56:30.275 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:56:30.275 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-14T08:56:30.275 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:56:30.275 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:56:30.275 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=2 2024-09-14T08:56:30.275 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=2 2024-09-14T08:56:30.275 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T08:56:30.291 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:56:30.291 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:56:30.291 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:56:30.291 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_16 2024-09-14T08:56:30.371 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_16'] 2024-09-14T08:56:30.371 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:56:30.371 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:56:30.374 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:56:30.374 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_16 2024-09-14T08:56:30.374 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:3 2024-09-14T08:56:30.374 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:56:30.375 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:56:30.375 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:3 cluster instance 2024-09-14T08:56:30.375 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:3 2024-09-14T08:56:30.375 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:56:30.375 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:56:30.375 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:56:30.375 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=3 2024-09-14T08:56:30.375 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-14T08:56:30.375 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:56:30.375 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:56:30.375 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=3 2024-09-14T08:56:30.375 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=3 2024-09-14T08:56:30.375 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:3 cluster1 2024-09-14T08:56:30.376 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:3 2024-09-14T08:56:30.376 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:56:30.376 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:56:30.376 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-14T08:56:30.376 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:3 2024-09-14T08:56:30.376 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:56:30.376 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:56:30.376 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:56:30.376 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=3 2024-09-14T08:56:30.376 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 3 = cluster1:3 ']' 2024-09-14T08:56:30.376 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:56:30.376 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:56:30.376 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=3 2024-09-14T08:56:30.376 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=3 2024-09-14T08:56:30.376 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-14T08:56:30.392 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T08:56:30.393 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T08:56:30.393 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T08:56:30.393 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_16 2024-09-14T08:56:30.472 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_16'] 2024-09-14T08:56:30.473 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:56:30.473 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:56:30.473 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:56:30.473 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sleep 4 2024-09-14T08:56:34.475 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T08:56:34.476 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ seq 0 3 2024-09-14T08:56:34.479 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:56:34.479 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_16 2024-09-14T08:56:34.479 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:0 2024-09-14T08:56:34.479 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:56:34.479 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:56:34.479 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T08:56:34.479 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:0 2024-09-14T08:56:34.479 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:56:34.479 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:56:34.479 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:56:34.479 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=0 2024-09-14T08:56:34.479 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T08:56:34.480 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:56:34.480 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:56:34.480 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=0 2024-09-14T08:56:34.480 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:56:34.480 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T08:56:34.480 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:0 2024-09-14T08:56:34.480 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:56:34.480 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:56:34.480 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T08:56:34.480 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:0 2024-09-14T08:56:34.480 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:56:34.480 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:56:34.480 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:56:34.480 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:56:34.481 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T08:56:34.481 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:56:34.481 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:56:34.481 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=0 2024-09-14T08:56:34.481 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=0 2024-09-14T08:56:34.481 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T08:56:34.500 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:56:34.500 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:56:34.500 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:56:34.500 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_16 2024-09-14T08:56:34.595 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ return 0 2024-09-14T08:56:34.596 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T08:56:34.596 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ echo ' "state": "Replaying"' 2024-09-14T08:56:34.596 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep -i Replaying 2024-09-14T08:56:34.597 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: "state": "Replaying" 2024-09-14T08:56:34.599 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_state=started 2024-09-14T08:56:34.599 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test started = started 2024-09-14T08:56:34.599 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:56:34.600 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_16 2024-09-14T08:56:34.600 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:56:34.600 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:56:34.600 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:34.600 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_16 2024-09-14T08:56:34.600 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = journal ']' 2024-09-14T08:56:34.600 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = snapshot ']' 2024-09-14T08:56:34.600 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_16 2024-09-14T08:56:34.600 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:56:34.600 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:56:34.600 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:34.600 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_16 2024-09-14T08:56:34.600 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster2-mirror-image_16.status 2024-09-14T08:56:34.600 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster2-mirror-image_16.status 2024-09-14T08:56:34.601 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:56:34.601 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_16.status 2024-09-14T08:56:34.601 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster1-mirror-image_16.status 2024-09-14T08:56:34.602 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster1-mirror-image_16.status 2024-09-14T08:56:34.602 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:56:34.602 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_status_log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_16.status 2024-09-14T08:56:34.602 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ mirror_image_snapshot cluster2 mirror image_16 2024-09-14T08:56:34.602 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:56:34.602 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:34.602 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_16 2024-09-14T08:56:34.602 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_16 2024-09-14T08:56:35.240 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Snapshot ID: 116 2024-09-14T08:56:35.246 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_16 /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_16.status 2024-09-14T08:56:35.246 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:56:35.246 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:35.246 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_16 2024-09-14T08:56:35.246 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_16.status 2024-09-14T08:56:35.246 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_16 --format xml 2024-09-14T08:56:35.247 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:56:35.293 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-14T08:56:35.295 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snapshot_id=116 2024-09-14T08:56:35.295 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:56:35.295 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:56:35.295 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.2 2024-09-14T08:56:35.496 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_16 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_16.status 2024-09-14T08:56:35.496 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:56:35.496 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:35.496 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_16 2024-09-14T08:56:35.497 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_16.status 2024-09-14T08:56:35.497 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_16 --format xml 2024-09-14T08:56:35.497 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:56:35.549 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:56:35.549 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:56:35.550 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:56:35.550 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:56:35.550 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:56:35.551 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:56:35.551 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 116 = '' 2024-09-14T08:56:35.551 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:56:35.551 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.4 2024-09-14T08:56:35.952 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_16 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_16.status 2024-09-14T08:56:35.952 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:56:35.952 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:35.952 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_16 2024-09-14T08:56:35.952 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_16.status 2024-09-14T08:56:35.952 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_16 --format xml 2024-09-14T08:56:35.952 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:56:36.001 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:56:36.001 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:56:36.003 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:56:36.003 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:56:36.003 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:56:36.003 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:56:36.003 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 116 = '' 2024-09-14T08:56:36.003 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:56:36.003 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.8 2024-09-14T08:56:36.804 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_16 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_16.status 2024-09-14T08:56:36.805 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:56:36.805 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:36.805 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_16 2024-09-14T08:56:36.805 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_16.status 2024-09-14T08:56:36.805 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_16 --format xml 2024-09-14T08:56:36.805 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:56:36.852 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:56:36.854 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=116 2024-09-14T08:56:36.854 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 116 = 116 2024-09-14T08:56:36.854 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:56:36.854 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snap_present cluster1 mirror image_16 snap 2024-09-14T08:56:36.854 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:56:36.855 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:36.855 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_16 2024-09-14T08:56:36.855 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:56:36.855 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:56:36.855 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T08:56:36.855 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:56:37.855 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 info mirror/image_16@snap 2024-09-14T08:56:37.904 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_16': 2024-09-14T08:56:37.905 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T08:56:37.905 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:56:37.905 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 2 2024-09-14T08:56:37.905 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 128732ca800c 2024-09-14T08:56:37.905 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.128732ca800c 2024-09-14T08:56:37.905 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:56:37.905 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering, non-primary 2024-09-14T08:56:37.905 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:56:37.905 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:56:37.905 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:47:38 2024 2024-09-14T08:56:37.905 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:47:38 2024 2024-09-14T08:56:37.905 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:47:38 2024 2024-09-14T08:56:37.906 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: protected: False 2024-09-14T08:56:37.906 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:56:37.906 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:56:37.906 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: 095f96c4-d4b7-4fe2-8ba2-0b31704d281f 2024-09-14T08:56:37.906 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: false 2024-09-14T08:56:37.909 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:56:37.909 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ compare_image_snaps mirror image_16 snap 2024-09-14T08:56:37.909 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:37.909 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_16 2024-09-14T08:56:37.909 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:56:37.909 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local ret=0 2024-09-14T08:56:37.909 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local rmt_export=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_16.export 2024-09-14T08:56:37.909 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local loc_export=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_16.export 2024-09-14T08:56:37.909 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_16.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_16.export 2024-09-14T08:56:37.910 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror export image_16@snap /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_16.export 2024-09-14T08:56:37.943 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.9817483583167869) > weight (0.25) 2024-09-14T08:56:37.943 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.9750125354467436) > weight (0.25) 2024-09-14T08:56:37.943 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.7990920786257671) > weight (0.25) 2024-09-14T08:56:37.943 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.3 with skip (0.6811787488590205) > weight (0.25) 2024-09-14T08:56:37.944 INFO:tasks.rbd_mirror_thrash:waiting for 83 secs before thrashing 2024-09-14T08:56:38.108 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T08:56:38.114 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 -p mirror export image_16@snap /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_16.export 2024-09-14T08:56:38.368 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T08:56:38.375 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cmp /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_16.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_16.export 2024-09-14T08:56:38.430 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_16.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_16.export 2024-09-14T08:56:38.461 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:56:38.461 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T08:56:38.462 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_17 2024-09-14T08:56:38.462 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_snap cluster2 mirror image_17 snap 2024-09-14T08:56:38.462 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:56:38.462 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:38.462 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_17 2024-09-14T08:56:38.462 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:56:38.462 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror snap create image_17@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T08:56:39.244 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_started cluster1 mirror image_17 2024-09-14T08:56:39.245 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:56:39.245 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:39.245 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_17 2024-09-14T08:56:39.245 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_state cluster1 mirror image_17 started 2024-09-14T08:56:39.245 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:56:39.245 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:39.245 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_17 2024-09-14T08:56:39.245 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=started 2024-09-14T08:56:39.245 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:56:39.245 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:56:39.245 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:56:40.245 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_replay_state cluster1 mirror image_17 started 2024-09-14T08:56:40.246 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:56:40.246 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:40.246 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_17 2024-09-14T08:56:40.246 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=started 2024-09-14T08:56:40.246 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_result 2024-09-14T08:56:40.246 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=stopped 2024-09-14T08:56:40.247 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_17 2024-09-14T08:56:40.247 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_instance=cluster1 2024-09-14T08:56:40.247 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:56:40.247 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ grep -i state 2024-09-14T08:56:40.247 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T08:56:40.247 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance=cluster1 2024-09-14T08:56:40.247 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local loop_instance 2024-09-14T08:56:40.247 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:56:40.247 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sleep 0 2024-09-14T08:56:40.248 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T08:56:40.248 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ seq 0 3 2024-09-14T08:56:40.250 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:56:40.250 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_17 2024-09-14T08:56:40.250 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:0 2024-09-14T08:56:40.250 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:56:40.250 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:56:40.250 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T08:56:40.251 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:0 2024-09-14T08:56:40.251 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:56:40.251 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:56:40.251 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:56:40.251 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=0 2024-09-14T08:56:40.251 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T08:56:40.251 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:56:40.251 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:56:40.251 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=0 2024-09-14T08:56:40.251 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:56:40.251 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T08:56:40.251 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:0 2024-09-14T08:56:40.251 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:56:40.251 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:56:40.252 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T08:56:40.252 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:0 2024-09-14T08:56:40.252 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:56:40.252 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:56:40.252 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:56:40.252 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:56:40.252 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T08:56:40.252 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:56:40.252 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:56:40.253 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=0 2024-09-14T08:56:40.253 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=0 2024-09-14T08:56:40.253 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T08:56:40.272 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:56:40.272 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:56:40.272 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:56:40.272 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_17 2024-09-14T08:56:40.361 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_17'] 2024-09-14T08:56:40.361 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:56:40.361 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:56:40.364 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:56:40.364 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_17 2024-09-14T08:56:40.364 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:1 2024-09-14T08:56:40.364 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:56:40.364 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:56:40.364 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T08:56:40.364 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:1 2024-09-14T08:56:40.364 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:56:40.365 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:56:40.365 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:56:40.365 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=1 2024-09-14T08:56:40.365 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T08:56:40.365 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:56:40.365 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:56:40.365 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=1 2024-09-14T08:56:40.365 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:56:40.365 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T08:56:40.365 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:1 2024-09-14T08:56:40.365 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:56:40.365 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:56:40.365 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T08:56:40.365 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:1 2024-09-14T08:56:40.366 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:56:40.366 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:56:40.366 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:56:40.366 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:56:40.366 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T08:56:40.366 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:56:40.366 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:56:40.366 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=1 2024-09-14T08:56:40.366 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=1 2024-09-14T08:56:40.366 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T08:56:40.383 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:56:40.383 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:56:40.383 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:56:40.383 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_17 2024-09-14T08:56:40.465 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_17'] 2024-09-14T08:56:40.465 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:56:40.465 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:56:40.468 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:56:40.468 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_17 2024-09-14T08:56:40.468 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:2 2024-09-14T08:56:40.469 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:56:40.469 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:56:40.469 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-14T08:56:40.469 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:2 2024-09-14T08:56:40.469 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:56:40.469 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:56:40.469 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:56:40.469 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=2 2024-09-14T08:56:40.469 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T08:56:40.469 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:56:40.469 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:56:40.469 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=2 2024-09-14T08:56:40.469 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:56:40.470 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-14T08:56:40.470 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:2 2024-09-14T08:56:40.470 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:56:40.470 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:56:40.470 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T08:56:40.470 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:2 2024-09-14T08:56:40.470 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:56:40.470 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:56:40.470 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:56:40.470 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:56:40.470 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-14T08:56:40.470 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:56:40.470 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:56:40.470 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=2 2024-09-14T08:56:40.471 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=2 2024-09-14T08:56:40.471 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T08:56:40.486 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:56:40.487 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:56:40.487 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:56:40.487 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_17 2024-09-14T08:56:40.575 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ return 0 2024-09-14T08:56:40.576 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T08:56:40.576 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ echo ' "state": "Replaying"' 2024-09-14T08:56:40.576 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep -i Replaying 2024-09-14T08:56:40.577 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: "state": "Replaying" 2024-09-14T08:56:40.579 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_state=started 2024-09-14T08:56:40.579 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test started = started 2024-09-14T08:56:40.579 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:56:40.579 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_17 2024-09-14T08:56:40.579 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:56:40.579 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:56:40.579 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:40.579 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_17 2024-09-14T08:56:40.579 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = journal ']' 2024-09-14T08:56:40.580 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = snapshot ']' 2024-09-14T08:56:40.580 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_17 2024-09-14T08:56:40.580 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:56:40.580 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:56:40.580 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:40.580 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_17 2024-09-14T08:56:40.580 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster2-mirror-image_17.status 2024-09-14T08:56:40.580 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster2-mirror-image_17.status 2024-09-14T08:56:40.580 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:56:40.580 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_17.status 2024-09-14T08:56:40.581 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster1-mirror-image_17.status 2024-09-14T08:56:40.581 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster1-mirror-image_17.status 2024-09-14T08:56:40.581 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:56:40.581 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_status_log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_17.status 2024-09-14T08:56:40.582 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ mirror_image_snapshot cluster2 mirror image_17 2024-09-14T08:56:40.582 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:56:40.582 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:40.582 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_17 2024-09-14T08:56:40.582 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_17 2024-09-14T08:56:41.256 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Snapshot ID: 118 2024-09-14T08:56:41.262 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_17 /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_17.status 2024-09-14T08:56:41.263 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:56:41.263 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:41.263 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_17 2024-09-14T08:56:41.263 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_17.status 2024-09-14T08:56:41.263 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_17 --format xml 2024-09-14T08:56:41.263 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:56:41.313 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-14T08:56:41.315 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snapshot_id=118 2024-09-14T08:56:41.315 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:56:41.315 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:56:41.315 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.2 2024-09-14T08:56:41.516 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_17 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_17.status 2024-09-14T08:56:41.516 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:56:41.517 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:41.517 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_17 2024-09-14T08:56:41.517 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_17.status 2024-09-14T08:56:41.517 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_17 --format xml 2024-09-14T08:56:41.517 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:56:41.568 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:56:41.904 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:56:41.905 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:56:41.905 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:56:41.906 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:56:41.906 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:56:41.906 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 118 = '' 2024-09-14T08:56:41.906 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:56:41.906 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.4 2024-09-14T08:56:41.971 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_17 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_17.status 2024-09-14T08:56:41.971 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:56:41.971 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:41.971 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_17 2024-09-14T08:56:41.971 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_17.status 2024-09-14T08:56:41.971 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_17 --format xml 2024-09-14T08:56:41.971 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:56:42.025 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:56:42.025 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:56:42.027 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:56:42.028 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:56:42.028 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:56:42.028 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:56:42.028 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 118 = '' 2024-09-14T08:56:42.028 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:56:42.029 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.8 2024-09-14T08:56:42.828 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_17 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_17.status 2024-09-14T08:56:42.829 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:56:42.829 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:42.829 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_17 2024-09-14T08:56:42.829 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_17.status 2024-09-14T08:56:42.829 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_17 --format xml 2024-09-14T08:56:42.829 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:56:42.877 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:56:42.879 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=118 2024-09-14T08:56:42.879 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 118 = 118 2024-09-14T08:56:42.879 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:56:42.879 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snap_present cluster1 mirror image_17 snap 2024-09-14T08:56:42.879 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:56:42.879 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:42.879 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_17 2024-09-14T08:56:42.879 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:56:42.880 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:56:42.880 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T08:56:42.880 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:56:43.880 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 info mirror/image_17@snap 2024-09-14T08:56:43.925 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_17': 2024-09-14T08:56:43.925 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T08:56:43.925 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:56:43.925 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 2 2024-09-14T08:56:43.925 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 10bd7a57c389 2024-09-14T08:56:43.925 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.10bd7a57c389 2024-09-14T08:56:43.925 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:56:43.926 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering, non-primary 2024-09-14T08:56:43.926 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:56:43.926 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:56:43.926 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:47:47 2024 2024-09-14T08:56:43.926 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:47:47 2024 2024-09-14T08:56:43.926 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:47:47 2024 2024-09-14T08:56:43.926 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: protected: False 2024-09-14T08:56:43.926 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:56:43.926 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:56:43.926 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: e90f529e-8f18-40e1-aa77-39aa510ca09b 2024-09-14T08:56:43.926 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: false 2024-09-14T08:56:43.928 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:56:43.929 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ compare_image_snaps mirror image_17 snap 2024-09-14T08:56:43.929 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:43.929 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_17 2024-09-14T08:56:43.929 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:56:43.929 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local ret=0 2024-09-14T08:56:43.929 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local rmt_export=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_17.export 2024-09-14T08:56:43.929 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local loc_export=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_17.export 2024-09-14T08:56:43.929 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_17.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_17.export 2024-09-14T08:56:43.929 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror export image_17@snap /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_17.export 2024-09-14T08:56:44.132 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T08:56:44.140 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 -p mirror export image_17@snap /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_17.export 2024-09-14T08:56:44.386 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T08:56:44.394 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cmp /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_17.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_17.export 2024-09-14T08:56:44.448 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_17.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_17.export 2024-09-14T08:56:44.479 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:56:44.479 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T08:56:44.479 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_18 2024-09-14T08:56:44.480 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_snap cluster2 mirror image_18 snap 2024-09-14T08:56:44.480 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:56:44.480 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:44.480 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_18 2024-09-14T08:56:44.480 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:56:44.480 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror snap create image_18@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T08:56:45.262 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_started cluster1 mirror image_18 2024-09-14T08:56:45.262 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:56:45.262 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:45.262 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_18 2024-09-14T08:56:45.262 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_state cluster1 mirror image_18 started 2024-09-14T08:56:45.263 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:56:45.263 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:45.263 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_18 2024-09-14T08:56:45.263 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=started 2024-09-14T08:56:45.263 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:56:45.263 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:56:45.263 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:56:46.263 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_replay_state cluster1 mirror image_18 started 2024-09-14T08:56:46.263 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:56:46.264 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:46.264 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_18 2024-09-14T08:56:46.264 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=started 2024-09-14T08:56:46.264 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_result 2024-09-14T08:56:46.264 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=stopped 2024-09-14T08:56:46.265 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_18 2024-09-14T08:56:46.265 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_instance=cluster1 2024-09-14T08:56:46.265 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:56:46.265 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ grep -i state 2024-09-14T08:56:46.265 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T08:56:46.265 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance=cluster1 2024-09-14T08:56:46.265 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local loop_instance 2024-09-14T08:56:46.265 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:56:46.265 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sleep 0 2024-09-14T08:56:46.265 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T08:56:46.266 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ seq 0 3 2024-09-14T08:56:46.268 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:56:46.268 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_18 2024-09-14T08:56:46.268 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:0 2024-09-14T08:56:46.269 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:56:46.269 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:56:46.269 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T08:56:46.269 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:0 2024-09-14T08:56:46.269 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:56:46.269 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:56:46.269 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:56:46.269 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=0 2024-09-14T08:56:46.269 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T08:56:46.269 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:56:46.269 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:56:46.269 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=0 2024-09-14T08:56:46.269 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:56:46.269 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T08:56:46.270 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:0 2024-09-14T08:56:46.270 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:56:46.270 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:56:46.270 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T08:56:46.270 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:0 2024-09-14T08:56:46.270 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:56:46.270 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:56:46.270 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:56:46.270 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:56:46.270 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T08:56:46.270 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:56:46.270 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:56:46.270 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=0 2024-09-14T08:56:46.270 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=0 2024-09-14T08:56:46.271 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T08:56:46.290 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:56:46.290 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:56:46.290 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:56:46.290 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_18 2024-09-14T08:56:46.378 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_18'] 2024-09-14T08:56:46.378 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:56:46.378 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:56:46.381 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:56:46.381 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_18 2024-09-14T08:56:46.381 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:1 2024-09-14T08:56:46.381 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:56:46.382 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:56:46.382 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T08:56:46.382 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:1 2024-09-14T08:56:46.382 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:56:46.382 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:56:46.382 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:56:46.382 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=1 2024-09-14T08:56:46.382 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T08:56:46.382 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:56:46.382 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:56:46.382 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=1 2024-09-14T08:56:46.382 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:56:46.382 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T08:56:46.382 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:1 2024-09-14T08:56:46.383 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:56:46.383 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:56:46.383 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T08:56:46.383 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:1 2024-09-14T08:56:46.383 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:56:46.383 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:56:46.383 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:56:46.383 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:56:46.383 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T08:56:46.383 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:56:46.383 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:56:46.383 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=1 2024-09-14T08:56:46.383 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=1 2024-09-14T08:56:46.383 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T08:56:46.400 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:56:46.400 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:56:46.401 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:56:46.401 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_18 2024-09-14T08:56:46.483 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_18'] 2024-09-14T08:56:46.483 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:56:46.483 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:56:46.486 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:56:46.486 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_18 2024-09-14T08:56:46.486 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:2 2024-09-14T08:56:46.486 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:56:46.486 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:56:46.486 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-14T08:56:46.486 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:2 2024-09-14T08:56:46.486 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:56:46.486 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:56:46.486 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:56:46.487 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=2 2024-09-14T08:56:46.487 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T08:56:46.487 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:56:46.487 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:56:46.487 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=2 2024-09-14T08:56:46.487 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:56:46.487 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-14T08:56:46.487 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:2 2024-09-14T08:56:46.487 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:56:46.487 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:56:46.487 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T08:56:46.487 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:2 2024-09-14T08:56:46.487 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:56:46.488 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:56:46.488 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:56:46.488 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:56:46.488 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-14T08:56:46.488 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:56:46.488 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:56:46.488 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=2 2024-09-14T08:56:46.488 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=2 2024-09-14T08:56:46.488 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T08:56:46.505 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:56:46.505 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:56:46.505 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:56:46.505 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_18 2024-09-14T08:56:46.586 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_18'] 2024-09-14T08:56:46.586 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:56:46.586 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:56:46.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:56:46.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_18 2024-09-14T08:56:46.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:3 2024-09-14T08:56:46.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:56:46.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:56:46.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:3 cluster instance 2024-09-14T08:56:46.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:3 2024-09-14T08:56:46.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:56:46.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:56:46.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:56:46.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=3 2024-09-14T08:56:46.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-14T08:56:46.591 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:56:46.591 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:56:46.591 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=3 2024-09-14T08:56:46.591 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=3 2024-09-14T08:56:46.591 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:3 cluster1 2024-09-14T08:56:46.591 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:3 2024-09-14T08:56:46.591 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:56:46.591 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:56:46.591 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-14T08:56:46.591 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:3 2024-09-14T08:56:46.591 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:56:46.591 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:56:46.591 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:56:46.591 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=3 2024-09-14T08:56:46.592 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 3 = cluster1:3 ']' 2024-09-14T08:56:46.592 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:56:46.592 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:56:46.592 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=3 2024-09-14T08:56:46.592 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=3 2024-09-14T08:56:46.592 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-14T08:56:46.609 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T08:56:46.609 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T08:56:46.609 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T08:56:46.609 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_18 2024-09-14T08:56:46.698 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ return 0 2024-09-14T08:56:46.698 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T08:56:46.698 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ echo ' "state": "Replaying"' 2024-09-14T08:56:46.698 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep -i Replaying 2024-09-14T08:56:46.699 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: "state": "Replaying" 2024-09-14T08:56:46.700 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_state=started 2024-09-14T08:56:46.700 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test started = started 2024-09-14T08:56:46.701 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:56:46.701 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_18 2024-09-14T08:56:46.701 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:56:46.701 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:56:46.701 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:46.701 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_18 2024-09-14T08:56:46.701 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = journal ']' 2024-09-14T08:56:46.701 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = snapshot ']' 2024-09-14T08:56:46.702 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_18 2024-09-14T08:56:46.702 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:56:46.702 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:56:46.702 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:46.702 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_18 2024-09-14T08:56:46.702 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster2-mirror-image_18.status 2024-09-14T08:56:46.702 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster2-mirror-image_18.status 2024-09-14T08:56:46.702 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:56:46.703 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_18.status 2024-09-14T08:56:46.703 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster1-mirror-image_18.status 2024-09-14T08:56:46.703 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster1-mirror-image_18.status 2024-09-14T08:56:46.703 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:56:46.703 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_status_log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_18.status 2024-09-14T08:56:46.704 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ mirror_image_snapshot cluster2 mirror image_18 2024-09-14T08:56:46.704 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:56:46.704 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:46.704 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_18 2024-09-14T08:56:46.704 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_18 2024-09-14T08:56:47.274 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Snapshot ID: 120 2024-09-14T08:56:47.281 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_18 /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_18.status 2024-09-14T08:56:47.281 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:56:47.281 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:47.281 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_18 2024-09-14T08:56:47.281 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_18.status 2024-09-14T08:56:47.281 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_18 --format xml 2024-09-14T08:56:47.282 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:56:47.328 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-14T08:56:47.330 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snapshot_id=120 2024-09-14T08:56:47.330 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:56:47.330 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:56:47.330 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.2 2024-09-14T08:56:47.531 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_18 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_18.status 2024-09-14T08:56:47.531 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:56:47.531 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:47.531 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_18 2024-09-14T08:56:47.531 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_18.status 2024-09-14T08:56:47.531 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_18 --format xml 2024-09-14T08:56:47.531 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:56:47.580 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:56:47.580 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:56:47.581 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:56:47.581 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:56:47.581 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:56:47.581 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:56:47.581 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 120 = '' 2024-09-14T08:56:47.582 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:56:47.582 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.4 2024-09-14T08:56:47.983 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_18 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_18.status 2024-09-14T08:56:47.983 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:56:47.983 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:47.983 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_18 2024-09-14T08:56:47.983 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_18.status 2024-09-14T08:56:47.983 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_18 --format xml 2024-09-14T08:56:47.983 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:56:48.032 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:56:48.033 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:56:48.035 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:56:48.035 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:56:48.035 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:56:48.035 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:56:48.035 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 120 = '' 2024-09-14T08:56:48.035 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:56:48.035 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.8 2024-09-14T08:56:48.836 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_18 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_18.status 2024-09-14T08:56:48.836 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:56:48.836 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:48.836 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_18 2024-09-14T08:56:48.836 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_18.status 2024-09-14T08:56:48.836 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_18 --format xml 2024-09-14T08:56:48.836 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:56:48.884 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:56:48.885 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=120 2024-09-14T08:56:48.886 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 120 = 120 2024-09-14T08:56:48.886 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:56:48.886 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snap_present cluster1 mirror image_18 snap 2024-09-14T08:56:48.886 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:56:48.886 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:48.886 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_18 2024-09-14T08:56:48.886 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:56:48.886 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:56:48.886 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T08:56:48.886 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:56:49.887 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 info mirror/image_18@snap 2024-09-14T08:56:49.936 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_18': 2024-09-14T08:56:49.937 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T08:56:49.937 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:56:49.937 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 2 2024-09-14T08:56:49.937 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 1189cddb7986 2024-09-14T08:56:49.937 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.1189cddb7986 2024-09-14T08:56:49.937 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:56:49.937 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering, non-primary 2024-09-14T08:56:49.937 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:56:49.937 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:56:49.937 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:48:01 2024 2024-09-14T08:56:49.937 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:48:01 2024 2024-09-14T08:56:49.937 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:48:01 2024 2024-09-14T08:56:49.937 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: protected: False 2024-09-14T08:56:49.938 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:56:49.938 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:56:49.938 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: 14306d77-7330-41b5-b70c-85c15b66b18d 2024-09-14T08:56:49.938 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: false 2024-09-14T08:56:49.941 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:56:49.941 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ compare_image_snaps mirror image_18 snap 2024-09-14T08:56:49.941 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:49.941 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_18 2024-09-14T08:56:49.941 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:56:49.941 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local ret=0 2024-09-14T08:56:49.941 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local rmt_export=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_18.export 2024-09-14T08:56:49.941 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local loc_export=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_18.export 2024-09-14T08:56:49.941 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_18.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_18.export 2024-09-14T08:56:49.942 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror export image_18@snap /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_18.export 2024-09-14T08:56:50.142 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T08:56:50.149 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 -p mirror export image_18@snap /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_18.export 2024-09-14T08:56:50.428 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T08:56:50.434 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cmp /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_18.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_18.export 2024-09-14T08:56:50.488 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_18.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_18.export 2024-09-14T08:56:50.520 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:56:50.520 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T08:56:50.520 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_19 2024-09-14T08:56:50.520 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_snap cluster2 mirror image_19 snap 2024-09-14T08:56:50.520 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:56:50.520 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:50.520 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_19 2024-09-14T08:56:50.520 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:56:50.520 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror snap create image_19@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T08:56:51.281 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_started cluster1 mirror image_19 2024-09-14T08:56:51.281 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:56:51.281 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:51.281 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_19 2024-09-14T08:56:51.281 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_state cluster1 mirror image_19 started 2024-09-14T08:56:51.281 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:56:51.281 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:51.281 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_19 2024-09-14T08:56:51.282 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=started 2024-09-14T08:56:51.282 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:56:51.282 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:56:51.282 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:56:52.282 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_replay_state cluster1 mirror image_19 started 2024-09-14T08:56:52.282 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:56:52.282 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:52.282 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_19 2024-09-14T08:56:52.282 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=started 2024-09-14T08:56:52.282 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_result 2024-09-14T08:56:52.282 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=stopped 2024-09-14T08:56:52.283 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_19 2024-09-14T08:56:52.284 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_instance=cluster1 2024-09-14T08:56:52.284 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:56:52.284 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ grep -i state 2024-09-14T08:56:52.284 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T08:56:52.284 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance=cluster1 2024-09-14T08:56:52.284 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local loop_instance 2024-09-14T08:56:52.284 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:56:52.284 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sleep 0 2024-09-14T08:56:52.284 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T08:56:52.284 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ seq 0 3 2024-09-14T08:56:52.286 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:56:52.286 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_19 2024-09-14T08:56:52.286 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:0 2024-09-14T08:56:52.286 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:56:52.287 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:56:52.287 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T08:56:52.287 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:0 2024-09-14T08:56:52.287 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:56:52.287 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:56:52.287 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:56:52.287 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=0 2024-09-14T08:56:52.287 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T08:56:52.287 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:56:52.287 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:56:52.287 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=0 2024-09-14T08:56:52.288 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:56:52.288 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T08:56:52.288 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:0 2024-09-14T08:56:52.288 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:56:52.288 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:56:52.288 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T08:56:52.288 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:0 2024-09-14T08:56:52.289 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:56:52.289 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:56:52.289 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:56:52.289 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:56:52.290 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T08:56:52.290 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:56:52.290 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:56:52.290 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=0 2024-09-14T08:56:52.290 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=0 2024-09-14T08:56:52.290 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T08:56:52.308 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:56:52.308 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:56:52.308 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:56:52.308 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_19 2024-09-14T08:56:52.392 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_19'] 2024-09-14T08:56:52.393 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:56:52.393 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:56:52.395 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:56:52.396 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_19 2024-09-14T08:56:52.396 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:1 2024-09-14T08:56:52.396 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:56:52.396 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:56:52.396 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T08:56:52.396 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:1 2024-09-14T08:56:52.396 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:56:52.396 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:56:52.396 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:56:52.396 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=1 2024-09-14T08:56:52.396 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T08:56:52.396 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:56:52.397 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:56:52.397 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=1 2024-09-14T08:56:52.397 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:56:52.397 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T08:56:52.397 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:1 2024-09-14T08:56:52.397 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:56:52.397 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:56:52.397 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T08:56:52.397 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:1 2024-09-14T08:56:52.397 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:56:52.397 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:56:52.397 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:56:52.397 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:56:52.397 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T08:56:52.398 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:56:52.398 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:56:52.398 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=1 2024-09-14T08:56:52.398 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=1 2024-09-14T08:56:52.398 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T08:56:52.416 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:56:52.416 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:56:52.416 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:56:52.416 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_19 2024-09-14T08:56:52.498 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_19'] 2024-09-14T08:56:52.499 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:56:52.499 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:56:52.505 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:56:52.505 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_19 2024-09-14T08:56:52.505 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:2 2024-09-14T08:56:52.506 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:56:52.506 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:56:52.506 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-14T08:56:52.506 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:2 2024-09-14T08:56:52.506 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:56:52.507 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:56:52.507 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:56:52.507 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=2 2024-09-14T08:56:52.507 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T08:56:52.507 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:56:52.508 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:56:52.508 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=2 2024-09-14T08:56:52.508 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:56:52.508 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-14T08:56:52.508 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:2 2024-09-14T08:56:52.508 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:56:52.508 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:56:52.509 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T08:56:52.509 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:2 2024-09-14T08:56:52.509 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:56:52.509 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:56:52.509 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:56:52.509 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:56:52.509 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-14T08:56:52.509 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:56:52.509 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:56:52.509 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=2 2024-09-14T08:56:52.509 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=2 2024-09-14T08:56:52.510 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T08:56:52.520 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:56:52.520 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:56:52.520 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:56:52.520 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_19 2024-09-14T08:56:52.602 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_19'] 2024-09-14T08:56:52.602 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:56:52.602 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:56:52.605 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:56:52.606 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_19 2024-09-14T08:56:52.606 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:3 2024-09-14T08:56:52.606 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:56:52.606 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:56:52.606 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:3 cluster instance 2024-09-14T08:56:52.606 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:3 2024-09-14T08:56:52.606 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:56:52.606 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:56:52.606 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:56:52.606 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=3 2024-09-14T08:56:52.606 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-14T08:56:52.606 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:56:52.606 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:56:52.607 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=3 2024-09-14T08:56:52.607 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=3 2024-09-14T08:56:52.607 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:3 cluster1 2024-09-14T08:56:52.607 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:3 2024-09-14T08:56:52.607 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:56:52.607 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:56:52.607 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-14T08:56:52.607 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:3 2024-09-14T08:56:52.607 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:56:52.607 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:56:52.607 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:56:52.607 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=3 2024-09-14T08:56:52.607 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 3 = cluster1:3 ']' 2024-09-14T08:56:52.607 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:56:52.608 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:56:52.608 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=3 2024-09-14T08:56:52.608 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=3 2024-09-14T08:56:52.608 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-14T08:56:52.624 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T08:56:52.624 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T08:56:52.624 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T08:56:52.624 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_19 2024-09-14T08:56:52.714 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ return 0 2024-09-14T08:56:52.714 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T08:56:52.715 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ echo ' "state": "Replaying"' 2024-09-14T08:56:52.715 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep -i Replaying 2024-09-14T08:56:52.715 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: "state": "Replaying" 2024-09-14T08:56:52.718 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_state=started 2024-09-14T08:56:52.718 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test started = started 2024-09-14T08:56:52.718 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:56:52.718 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_19 2024-09-14T08:56:52.718 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:56:52.718 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:56:52.718 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:52.719 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_19 2024-09-14T08:56:52.719 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = journal ']' 2024-09-14T08:56:52.719 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = snapshot ']' 2024-09-14T08:56:52.719 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_19 2024-09-14T08:56:52.719 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:56:52.719 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:56:52.719 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:52.719 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_19 2024-09-14T08:56:52.719 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster2-mirror-image_19.status 2024-09-14T08:56:52.719 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster2-mirror-image_19.status 2024-09-14T08:56:52.719 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:56:52.719 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_19.status 2024-09-14T08:56:52.720 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster1-mirror-image_19.status 2024-09-14T08:56:52.720 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster1-mirror-image_19.status 2024-09-14T08:56:52.720 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:56:52.721 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_status_log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_19.status 2024-09-14T08:56:52.721 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ mirror_image_snapshot cluster2 mirror image_19 2024-09-14T08:56:52.721 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:56:52.721 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:52.721 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_19 2024-09-14T08:56:52.721 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_19 2024-09-14T08:56:53.282 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Snapshot ID: 122 2024-09-14T08:56:53.288 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_19 /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_19.status 2024-09-14T08:56:53.289 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:56:53.289 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:53.289 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_19 2024-09-14T08:56:53.289 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_19.status 2024-09-14T08:56:53.289 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_19 --format xml 2024-09-14T08:56:53.289 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:56:53.336 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-14T08:56:53.337 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snapshot_id=122 2024-09-14T08:56:53.337 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:56:53.338 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:56:53.338 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.2 2024-09-14T08:56:53.539 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_19 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_19.status 2024-09-14T08:56:53.539 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:56:53.539 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:53.539 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_19 2024-09-14T08:56:53.539 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_19.status 2024-09-14T08:56:53.539 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_19 --format xml 2024-09-14T08:56:53.539 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:56:53.588 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:56:53.588 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:56:53.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:56:53.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:56:53.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:56:53.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:56:53.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 122 = '' 2024-09-14T08:56:53.591 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:56:53.591 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.4 2024-09-14T08:56:53.991 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_19 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_19.status 2024-09-14T08:56:53.991 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:56:53.991 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:53.991 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_19 2024-09-14T08:56:53.991 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_19.status 2024-09-14T08:56:53.991 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_19 --format xml 2024-09-14T08:56:53.992 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:56:54.043 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:56:54.043 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:56:54.045 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:56:54.045 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:56:54.045 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:56:54.046 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:56:54.046 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 122 = '' 2024-09-14T08:56:54.046 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:56:54.046 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.8 2024-09-14T08:56:54.847 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_19 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_19.status 2024-09-14T08:56:54.847 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:56:54.847 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:54.847 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_19 2024-09-14T08:56:54.847 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_19.status 2024-09-14T08:56:54.847 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_19 --format xml 2024-09-14T08:56:54.847 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:56:54.895 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:56:54.897 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=122 2024-09-14T08:56:54.897 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 122 = 122 2024-09-14T08:56:54.897 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:56:54.897 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snap_present cluster1 mirror image_19 snap 2024-09-14T08:56:54.897 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:56:54.897 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:54.897 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_19 2024-09-14T08:56:54.897 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:56:54.897 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:56:54.898 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T08:56:54.898 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:56:55.898 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 info mirror/image_19@snap 2024-09-14T08:56:55.945 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_19': 2024-09-14T08:56:55.945 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T08:56:55.945 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:56:55.945 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 2 2024-09-14T08:56:55.946 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 127ecb6b408d 2024-09-14T08:56:55.946 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.127ecb6b408d 2024-09-14T08:56:55.946 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:56:55.946 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering, non-primary 2024-09-14T08:56:55.946 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:56:55.946 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:56:55.946 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:48:12 2024 2024-09-14T08:56:55.946 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:48:12 2024 2024-09-14T08:56:55.946 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:48:12 2024 2024-09-14T08:56:55.946 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: protected: False 2024-09-14T08:56:55.947 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:56:55.947 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:56:55.947 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: 72324075-a734-4269-8742-62ebe1d962b0 2024-09-14T08:56:55.947 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: false 2024-09-14T08:56:55.950 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:56:55.950 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ compare_image_snaps mirror image_19 snap 2024-09-14T08:56:55.950 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:55.950 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_19 2024-09-14T08:56:55.950 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:56:55.950 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local ret=0 2024-09-14T08:56:55.951 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local rmt_export=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_19.export 2024-09-14T08:56:55.951 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local loc_export=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_19.export 2024-09-14T08:56:55.951 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_19.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_19.export 2024-09-14T08:56:55.951 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror export image_19@snap /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_19.export 2024-09-14T08:56:56.145 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T08:56:56.152 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 -p mirror export image_19@snap /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_19.export 2024-09-14T08:56:56.404 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T08:56:56.411 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cmp /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_19.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_19.export 2024-09-14T08:56:56.466 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_19.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_19.export 2024-09-14T08:56:56.497 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:56:56.497 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T08:56:56.497 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_20 2024-09-14T08:56:56.497 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_snap cluster2 mirror image_20 snap 2024-09-14T08:56:56.497 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:56:56.497 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:56.497 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_20 2024-09-14T08:56:56.497 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:56:56.497 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror snap create image_20@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T08:56:57.279 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_started cluster1 mirror image_20 2024-09-14T08:56:57.279 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:56:57.279 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:57.279 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_20 2024-09-14T08:56:57.279 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_state cluster1 mirror image_20 started 2024-09-14T08:56:57.279 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:56:57.280 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:57.280 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_20 2024-09-14T08:56:57.280 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=started 2024-09-14T08:56:57.280 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:56:57.280 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:56:57.280 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:56:58.280 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_replay_state cluster1 mirror image_20 started 2024-09-14T08:56:58.281 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:56:58.281 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:58.281 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_20 2024-09-14T08:56:58.281 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=started 2024-09-14T08:56:58.281 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_result 2024-09-14T08:56:58.281 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=stopped 2024-09-14T08:56:58.281 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_20 2024-09-14T08:56:58.281 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_instance=cluster1 2024-09-14T08:56:58.281 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ grep -i state 2024-09-14T08:56:58.281 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:56:58.281 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T08:56:58.281 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance=cluster1 2024-09-14T08:56:58.282 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local loop_instance 2024-09-14T08:56:58.282 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:56:58.282 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sleep 0 2024-09-14T08:56:58.284 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T08:56:58.284 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ seq 0 3 2024-09-14T08:56:58.284 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:56:58.284 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_20 2024-09-14T08:56:58.284 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:0 2024-09-14T08:56:58.284 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:56:58.284 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:56:58.284 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T08:56:58.284 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:0 2024-09-14T08:56:58.284 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:56:58.284 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:56:58.284 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:56:58.284 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=0 2024-09-14T08:56:58.285 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T08:56:58.285 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:56:58.285 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:56:58.285 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=0 2024-09-14T08:56:58.285 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:56:58.285 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T08:56:58.285 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:0 2024-09-14T08:56:58.285 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:56:58.285 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:56:58.285 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T08:56:58.285 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:0 2024-09-14T08:56:58.285 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:56:58.285 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:56:58.286 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:56:58.287 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:56:58.287 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T08:56:58.287 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:56:58.287 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:56:58.287 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=0 2024-09-14T08:56:58.287 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=0 2024-09-14T08:56:58.287 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T08:56:58.305 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:56:58.305 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:56:58.306 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:56:58.306 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_20 2024-09-14T08:56:58.391 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_20'] 2024-09-14T08:56:58.392 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:56:58.392 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:56:58.395 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:56:58.395 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_20 2024-09-14T08:56:58.395 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:1 2024-09-14T08:56:58.395 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:56:58.395 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:56:58.395 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T08:56:58.395 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:1 2024-09-14T08:56:58.395 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:56:58.395 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:56:58.395 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:56:58.395 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=1 2024-09-14T08:56:58.395 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T08:56:58.396 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:56:58.396 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:56:58.396 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=1 2024-09-14T08:56:58.396 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:56:58.397 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T08:56:58.397 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:1 2024-09-14T08:56:58.397 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:56:58.397 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:56:58.397 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T08:56:58.397 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:1 2024-09-14T08:56:58.397 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:56:58.397 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:56:58.397 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:56:58.397 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:56:58.397 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T08:56:58.397 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:56:58.397 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:56:58.397 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=1 2024-09-14T08:56:58.398 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=1 2024-09-14T08:56:58.398 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T08:56:58.413 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:56:58.413 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:56:58.413 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:56:58.413 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_20 2024-09-14T08:56:58.502 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ return 0 2024-09-14T08:56:58.502 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T08:56:58.503 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ echo ' "state": "Replaying"' 2024-09-14T08:56:58.503 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep -i Replaying 2024-09-14T08:56:58.504 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: "state": "Replaying" 2024-09-14T08:56:58.505 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_state=started 2024-09-14T08:56:58.505 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test started = started 2024-09-14T08:56:58.505 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:56:58.505 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_20 2024-09-14T08:56:58.505 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:56:58.505 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:56:58.505 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:58.506 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_20 2024-09-14T08:56:58.506 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = journal ']' 2024-09-14T08:56:58.506 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = snapshot ']' 2024-09-14T08:56:58.506 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_20 2024-09-14T08:56:58.506 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:56:58.506 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:56:58.506 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:58.506 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_20 2024-09-14T08:56:58.506 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster2-mirror-image_20.status 2024-09-14T08:56:58.506 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster2-mirror-image_20.status 2024-09-14T08:56:58.506 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:56:58.507 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_20.status 2024-09-14T08:56:58.507 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster1-mirror-image_20.status 2024-09-14T08:56:58.508 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster1-mirror-image_20.status 2024-09-14T08:56:58.508 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:56:58.508 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_status_log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_20.status 2024-09-14T08:56:58.508 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ mirror_image_snapshot cluster2 mirror image_20 2024-09-14T08:56:58.508 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:56:58.508 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:58.508 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_20 2024-09-14T08:56:58.508 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_20 2024-09-14T08:56:59.288 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Snapshot ID: 124 2024-09-14T08:56:59.294 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_20 /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_20.status 2024-09-14T08:56:59.294 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:56:59.294 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:59.294 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_20 2024-09-14T08:56:59.294 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_20.status 2024-09-14T08:56:59.295 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_20 --format xml 2024-09-14T08:56:59.295 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:56:59.342 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-14T08:56:59.343 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snapshot_id=124 2024-09-14T08:56:59.344 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:56:59.344 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:56:59.344 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.2 2024-09-14T08:56:59.545 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_20 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_20.status 2024-09-14T08:56:59.545 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:56:59.546 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:59.546 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_20 2024-09-14T08:56:59.546 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_20.status 2024-09-14T08:56:59.546 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_20 --format xml 2024-09-14T08:56:59.546 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:56:59.594 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:56:59.594 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:56:59.595 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:56:59.595 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:56:59.595 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:56:59.596 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:56:59.596 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 124 = '' 2024-09-14T08:56:59.596 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:56:59.596 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.4 2024-09-14T08:56:59.997 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_20 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_20.status 2024-09-14T08:56:59.997 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:56:59.997 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:56:59.997 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_20 2024-09-14T08:56:59.997 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_20.status 2024-09-14T08:56:59.997 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_20 --format xml 2024-09-14T08:56:59.997 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:57:00.046 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:57:00.046 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:57:00.048 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:57:00.048 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:57:00.048 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:57:00.048 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:57:00.049 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 124 = '' 2024-09-14T08:57:00.049 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:57:00.049 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.8 2024-09-14T08:57:00.850 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_20 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_20.status 2024-09-14T08:57:00.850 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:57:00.850 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:00.850 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_20 2024-09-14T08:57:00.850 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_20.status 2024-09-14T08:57:00.850 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_20 --format xml 2024-09-14T08:57:00.850 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:57:00.897 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:57:00.899 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=124 2024-09-14T08:57:00.899 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 124 = 124 2024-09-14T08:57:00.899 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:57:00.899 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snap_present cluster1 mirror image_20 snap 2024-09-14T08:57:00.900 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:57:00.900 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:00.900 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_20 2024-09-14T08:57:00.900 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:57:00.900 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:57:00.900 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T08:57:00.900 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:57:01.900 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 info mirror/image_20@snap 2024-09-14T08:57:01.950 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_20': 2024-09-14T08:57:01.950 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T08:57:01.950 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:57:01.950 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 2 2024-09-14T08:57:01.950 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 127e1819d469 2024-09-14T08:57:01.950 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.127e1819d469 2024-09-14T08:57:01.950 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:57:01.950 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering, non-primary 2024-09-14T08:57:01.950 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:57:01.950 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:57:01.950 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:48:48 2024 2024-09-14T08:57:01.950 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:48:48 2024 2024-09-14T08:57:01.950 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:48:48 2024 2024-09-14T08:57:01.951 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: protected: False 2024-09-14T08:57:01.951 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:57:01.951 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:57:01.951 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: f5c8b83b-151b-465c-b8b7-1ea8268e2846 2024-09-14T08:57:01.951 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: false 2024-09-14T08:57:01.954 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:57:01.954 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ compare_image_snaps mirror image_20 snap 2024-09-14T08:57:01.954 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:01.954 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_20 2024-09-14T08:57:01.954 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:57:01.954 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local ret=0 2024-09-14T08:57:01.954 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local rmt_export=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_20.export 2024-09-14T08:57:01.955 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local loc_export=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_20.export 2024-09-14T08:57:01.955 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_20.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_20.export 2024-09-14T08:57:01.955 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror export image_20@snap /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_20.export 2024-09-14T08:57:02.155 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T08:57:02.163 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 -p mirror export image_20@snap /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_20.export 2024-09-14T08:57:02.425 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T08:57:02.432 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cmp /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_20.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_20.export 2024-09-14T08:57:02.486 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_20.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_20.export 2024-09-14T08:57:02.518 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:57:02.518 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T08:57:02.518 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_21 2024-09-14T08:57:02.518 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_snap cluster2 mirror image_21 snap 2024-09-14T08:57:02.518 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:57:02.519 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:02.519 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_21 2024-09-14T08:57:02.519 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:57:02.519 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror snap create image_21@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T08:57:03.293 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_started cluster1 mirror image_21 2024-09-14T08:57:03.293 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:57:03.293 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:03.294 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_21 2024-09-14T08:57:03.294 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_state cluster1 mirror image_21 started 2024-09-14T08:57:03.294 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:57:03.294 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:03.294 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_21 2024-09-14T08:57:03.294 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=started 2024-09-14T08:57:03.294 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:57:03.294 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:57:03.294 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:57:04.294 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_replay_state cluster1 mirror image_21 started 2024-09-14T08:57:04.294 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:57:04.295 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:04.295 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_21 2024-09-14T08:57:04.295 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=started 2024-09-14T08:57:04.295 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_result 2024-09-14T08:57:04.295 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=stopped 2024-09-14T08:57:04.296 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_21 2024-09-14T08:57:04.296 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ grep -i state 2024-09-14T08:57:04.296 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_instance=cluster1 2024-09-14T08:57:04.296 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:57:04.296 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T08:57:04.297 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance=cluster1 2024-09-14T08:57:04.297 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local loop_instance 2024-09-14T08:57:04.297 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:57:04.297 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sleep 0 2024-09-14T08:57:04.297 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T08:57:04.302 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ seq 0 3 2024-09-14T08:57:04.302 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:57:04.302 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_21 2024-09-14T08:57:04.302 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:0 2024-09-14T08:57:04.302 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:57:04.302 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:57:04.302 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T08:57:04.303 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:0 2024-09-14T08:57:04.303 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:57:04.303 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:57:04.303 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:57:04.303 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=0 2024-09-14T08:57:04.303 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T08:57:04.303 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:57:04.303 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:57:04.303 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=0 2024-09-14T08:57:04.303 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:57:04.303 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T08:57:04.303 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:0 2024-09-14T08:57:04.303 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:57:04.303 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:57:04.303 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T08:57:04.304 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:0 2024-09-14T08:57:04.304 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:57:04.304 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:57:04.304 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:57:04.304 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:57:04.304 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T08:57:04.304 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:57:04.304 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:57:04.304 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=0 2024-09-14T08:57:04.304 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=0 2024-09-14T08:57:04.304 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T08:57:04.321 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:57:04.321 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:57:04.321 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:57:04.321 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_21 2024-09-14T08:57:04.409 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_21'] 2024-09-14T08:57:04.410 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:57:04.410 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:57:04.413 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:57:04.413 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_21 2024-09-14T08:57:04.413 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:1 2024-09-14T08:57:04.413 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:57:04.413 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:57:04.413 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T08:57:04.414 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:1 2024-09-14T08:57:04.414 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:57:04.414 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:57:04.414 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:57:04.414 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=1 2024-09-14T08:57:04.414 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T08:57:04.414 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:57:04.414 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:57:04.414 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=1 2024-09-14T08:57:04.414 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:57:04.414 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T08:57:04.414 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:1 2024-09-14T08:57:04.414 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:57:04.414 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:57:04.414 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T08:57:04.415 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:1 2024-09-14T08:57:04.415 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:57:04.415 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:57:04.415 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:57:04.415 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:57:04.415 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T08:57:04.415 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:57:04.415 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:57:04.415 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=1 2024-09-14T08:57:04.415 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=1 2024-09-14T08:57:04.415 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T08:57:04.432 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:57:04.432 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:57:04.432 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:57:04.432 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_21 2024-09-14T08:57:04.522 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ return 0 2024-09-14T08:57:04.522 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T08:57:04.522 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ echo ' "state": "Replaying"' 2024-09-14T08:57:04.522 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep -i Replaying 2024-09-14T08:57:04.523 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: "state": "Replaying" 2024-09-14T08:57:04.525 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_state=started 2024-09-14T08:57:04.525 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test started = started 2024-09-14T08:57:04.525 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:57:04.525 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_21 2024-09-14T08:57:04.525 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:57:04.525 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:57:04.525 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:04.525 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_21 2024-09-14T08:57:04.525 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = journal ']' 2024-09-14T08:57:04.525 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = snapshot ']' 2024-09-14T08:57:04.525 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_21 2024-09-14T08:57:04.525 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:57:04.525 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:57:04.525 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:04.526 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_21 2024-09-14T08:57:04.526 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster2-mirror-image_21.status 2024-09-14T08:57:04.526 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster2-mirror-image_21.status 2024-09-14T08:57:04.526 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:57:04.527 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_21.status 2024-09-14T08:57:04.527 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster1-mirror-image_21.status 2024-09-14T08:57:04.527 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster1-mirror-image_21.status 2024-09-14T08:57:04.527 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:57:04.528 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_status_log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_21.status 2024-09-14T08:57:04.528 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ mirror_image_snapshot cluster2 mirror image_21 2024-09-14T08:57:04.528 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:57:04.528 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:04.528 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_21 2024-09-14T08:57:04.528 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_21 2024-09-14T08:57:05.300 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Snapshot ID: 126 2024-09-14T08:57:05.307 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_21 /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_21.status 2024-09-14T08:57:05.308 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:57:05.308 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:05.308 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_21 2024-09-14T08:57:05.308 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_21.status 2024-09-14T08:57:05.308 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_21 --format xml 2024-09-14T08:57:05.308 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:57:05.355 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-14T08:57:05.357 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snapshot_id=126 2024-09-14T08:57:05.357 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:57:05.357 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:57:05.357 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.2 2024-09-14T08:57:05.558 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_21 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_21.status 2024-09-14T08:57:05.559 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:57:05.559 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:05.559 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_21 2024-09-14T08:57:05.559 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_21.status 2024-09-14T08:57:05.559 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_21 --format xml 2024-09-14T08:57:05.559 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:57:05.607 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:57:05.608 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:57:05.609 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:57:05.609 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:57:05.609 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:57:05.610 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:57:05.610 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 126 = '' 2024-09-14T08:57:05.610 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:57:05.610 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.4 2024-09-14T08:57:06.011 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_21 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_21.status 2024-09-14T08:57:06.011 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:57:06.011 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:06.011 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_21 2024-09-14T08:57:06.011 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_21.status 2024-09-14T08:57:06.011 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_21 --format xml 2024-09-14T08:57:06.011 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:57:06.061 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:57:06.062 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:57:06.063 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:57:06.063 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:57:06.063 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:57:06.063 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:57:06.063 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 126 = '' 2024-09-14T08:57:06.064 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:57:06.064 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.8 2024-09-14T08:57:06.865 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_21 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_21.status 2024-09-14T08:57:06.865 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:57:06.865 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:06.865 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_21 2024-09-14T08:57:06.865 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_21.status 2024-09-14T08:57:06.865 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_21 --format xml 2024-09-14T08:57:06.865 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:57:06.913 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:57:06.915 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=126 2024-09-14T08:57:06.915 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 126 = 126 2024-09-14T08:57:06.915 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:57:06.915 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snap_present cluster1 mirror image_21 snap 2024-09-14T08:57:06.915 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:57:06.916 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:06.916 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_21 2024-09-14T08:57:06.916 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:57:06.916 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:57:06.916 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T08:57:06.916 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:57:07.916 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 info mirror/image_21@snap 2024-09-14T08:57:07.960 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_21': 2024-09-14T08:57:07.960 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T08:57:07.960 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:57:07.960 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 2 2024-09-14T08:57:07.960 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 12964856e709 2024-09-14T08:57:07.960 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.12964856e709 2024-09-14T08:57:07.960 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:57:07.960 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering, non-primary 2024-09-14T08:57:07.961 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:57:07.961 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:57:07.961 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:48:33 2024 2024-09-14T08:57:07.961 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:48:33 2024 2024-09-14T08:57:07.961 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:48:33 2024 2024-09-14T08:57:07.961 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: protected: False 2024-09-14T08:57:07.961 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:57:07.961 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:57:07.961 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: 75208eea-1f60-4048-a017-3885fa315408 2024-09-14T08:57:07.961 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: false 2024-09-14T08:57:07.963 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:57:07.964 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ compare_image_snaps mirror image_21 snap 2024-09-14T08:57:07.964 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:07.964 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_21 2024-09-14T08:57:07.964 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:57:07.964 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local ret=0 2024-09-14T08:57:07.964 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local rmt_export=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_21.export 2024-09-14T08:57:07.964 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local loc_export=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_21.export 2024-09-14T08:57:07.964 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_21.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_21.export 2024-09-14T08:57:07.964 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror export image_21@snap /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_21.export 2024-09-14T08:57:08.148 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T08:57:08.156 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 -p mirror export image_21@snap /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_21.export 2024-09-14T08:57:08.395 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T08:57:08.403 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cmp /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_21.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_21.export 2024-09-14T08:57:08.457 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_21.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_21.export 2024-09-14T08:57:08.487 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:57:08.487 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T08:57:08.487 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_22 2024-09-14T08:57:08.488 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_snap cluster2 mirror image_22 snap 2024-09-14T08:57:08.488 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:57:08.488 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:08.488 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_22 2024-09-14T08:57:08.488 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:57:08.488 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror snap create image_22@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T08:57:09.298 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_started cluster1 mirror image_22 2024-09-14T08:57:09.299 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:57:09.299 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:09.299 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_22 2024-09-14T08:57:09.299 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_state cluster1 mirror image_22 started 2024-09-14T08:57:09.299 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:57:09.299 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:09.299 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_22 2024-09-14T08:57:09.299 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=started 2024-09-14T08:57:09.299 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:57:09.299 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:57:09.299 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:57:10.300 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_replay_state cluster1 mirror image_22 started 2024-09-14T08:57:10.300 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:57:10.300 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:10.300 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_22 2024-09-14T08:57:10.300 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=started 2024-09-14T08:57:10.300 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_result 2024-09-14T08:57:10.300 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=stopped 2024-09-14T08:57:10.301 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_22 2024-09-14T08:57:10.301 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_instance=cluster1 2024-09-14T08:57:10.301 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ grep -i state 2024-09-14T08:57:10.302 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:57:10.302 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T08:57:10.302 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance=cluster1 2024-09-14T08:57:10.302 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local loop_instance 2024-09-14T08:57:10.302 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:57:10.302 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sleep 0 2024-09-14T08:57:10.302 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T08:57:10.302 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ seq 0 3 2024-09-14T08:57:10.304 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:57:10.304 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_22 2024-09-14T08:57:10.304 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:0 2024-09-14T08:57:10.304 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:57:10.304 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:57:10.305 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T08:57:10.305 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:0 2024-09-14T08:57:10.305 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:57:10.305 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:57:10.305 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:57:10.305 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=0 2024-09-14T08:57:10.305 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T08:57:10.305 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:57:10.305 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:57:10.305 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=0 2024-09-14T08:57:10.305 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:57:10.305 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T08:57:10.305 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:0 2024-09-14T08:57:10.307 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:57:10.307 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:57:10.307 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T08:57:10.307 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:0 2024-09-14T08:57:10.307 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:57:10.307 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:57:10.307 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:57:10.307 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:57:10.307 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T08:57:10.307 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:57:10.308 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:57:10.308 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=0 2024-09-14T08:57:10.308 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=0 2024-09-14T08:57:10.308 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T08:57:10.328 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:57:10.328 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:57:10.328 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:57:10.328 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_22 2024-09-14T08:57:10.417 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_22'] 2024-09-14T08:57:10.417 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:57:10.417 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:57:10.420 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:57:10.420 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_22 2024-09-14T08:57:10.420 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:1 2024-09-14T08:57:10.420 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:57:10.420 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:57:10.420 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T08:57:10.420 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:1 2024-09-14T08:57:10.420 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:57:10.421 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:57:10.421 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:57:10.421 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=1 2024-09-14T08:57:10.421 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T08:57:10.421 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:57:10.421 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:57:10.421 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=1 2024-09-14T08:57:10.421 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:57:10.421 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T08:57:10.421 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:1 2024-09-14T08:57:10.421 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:57:10.422 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:57:10.422 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T08:57:10.422 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:1 2024-09-14T08:57:10.422 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:57:10.422 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:57:10.422 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:57:10.422 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:57:10.422 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T08:57:10.422 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:57:10.422 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:57:10.422 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=1 2024-09-14T08:57:10.422 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=1 2024-09-14T08:57:10.422 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T08:57:10.440 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:57:10.440 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:57:10.441 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:57:10.441 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_22 2024-09-14T08:57:10.528 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ return 0 2024-09-14T08:57:10.528 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T08:57:10.529 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ echo ' "state": "Replaying"' 2024-09-14T08:57:10.529 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep -i Replaying 2024-09-14T08:57:10.530 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: "state": "Replaying" 2024-09-14T08:57:10.531 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_state=started 2024-09-14T08:57:10.531 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test started = started 2024-09-14T08:57:10.531 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:57:10.531 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_22 2024-09-14T08:57:10.531 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:57:10.531 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:57:10.532 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:10.532 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_22 2024-09-14T08:57:10.532 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = journal ']' 2024-09-14T08:57:10.532 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = snapshot ']' 2024-09-14T08:57:10.532 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_22 2024-09-14T08:57:10.532 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:57:10.532 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:57:10.532 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:10.532 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_22 2024-09-14T08:57:10.532 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster2-mirror-image_22.status 2024-09-14T08:57:10.532 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster2-mirror-image_22.status 2024-09-14T08:57:10.532 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:57:10.533 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_22.status 2024-09-14T08:57:10.533 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster1-mirror-image_22.status 2024-09-14T08:57:10.533 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster1-mirror-image_22.status 2024-09-14T08:57:10.533 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:57:10.534 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_status_log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_22.status 2024-09-14T08:57:10.534 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ mirror_image_snapshot cluster2 mirror image_22 2024-09-14T08:57:10.534 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:57:10.534 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:10.534 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_22 2024-09-14T08:57:10.534 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_22 2024-09-14T08:57:11.304 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Snapshot ID: 128 2024-09-14T08:57:11.311 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_22 /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_22.status 2024-09-14T08:57:11.311 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:57:11.311 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:11.311 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_22 2024-09-14T08:57:11.311 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_22.status 2024-09-14T08:57:11.311 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_22 --format xml 2024-09-14T08:57:11.311 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:57:11.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-14T08:57:11.363 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snapshot_id=128 2024-09-14T08:57:11.363 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:57:11.364 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:57:11.364 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.2 2024-09-14T08:57:11.565 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_22 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_22.status 2024-09-14T08:57:11.565 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:57:11.565 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:11.565 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_22 2024-09-14T08:57:11.565 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_22.status 2024-09-14T08:57:11.565 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_22 --format xml 2024-09-14T08:57:11.566 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:57:11.615 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:57:11.615 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:57:11.616 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:57:11.616 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:57:11.616 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:57:11.617 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:57:11.617 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 128 = '' 2024-09-14T08:57:11.617 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:57:11.617 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.4 2024-09-14T08:57:12.018 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_22 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_22.status 2024-09-14T08:57:12.018 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:57:12.018 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:12.018 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_22 2024-09-14T08:57:12.019 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_22.status 2024-09-14T08:57:12.019 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_22 --format xml 2024-09-14T08:57:12.019 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:57:12.068 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:57:12.068 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:57:12.070 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:57:12.070 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:57:12.070 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:57:12.070 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:57:12.070 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 128 = '' 2024-09-14T08:57:12.070 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:57:12.070 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.8 2024-09-14T08:57:12.872 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_22 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_22.status 2024-09-14T08:57:12.872 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:57:12.872 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:12.873 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_22 2024-09-14T08:57:12.873 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_22.status 2024-09-14T08:57:12.873 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_22 --format xml 2024-09-14T08:57:12.873 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:57:12.919 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:57:12.922 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=128 2024-09-14T08:57:12.922 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 128 = 128 2024-09-14T08:57:12.922 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:57:12.922 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snap_present cluster1 mirror image_22 snap 2024-09-14T08:57:12.922 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:57:12.923 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:12.923 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_22 2024-09-14T08:57:12.923 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:57:12.923 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:57:12.923 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T08:57:12.923 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:57:13.922 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 info mirror/image_22@snap 2024-09-14T08:57:13.967 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_22': 2024-09-14T08:57:13.967 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T08:57:13.967 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:57:13.967 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 2 2024-09-14T08:57:13.968 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 1296261563c4 2024-09-14T08:57:13.968 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.1296261563c4 2024-09-14T08:57:13.968 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:57:13.968 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering, non-primary 2024-09-14T08:57:13.968 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:57:13.968 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:57:13.968 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:48:39 2024 2024-09-14T08:57:13.968 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:48:39 2024 2024-09-14T08:57:13.968 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:48:39 2024 2024-09-14T08:57:13.968 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: protected: False 2024-09-14T08:57:13.968 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:57:13.968 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:57:13.968 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: 83c2e42b-5fd9-42ba-b3f2-2e9c690a99bf 2024-09-14T08:57:13.968 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: false 2024-09-14T08:57:13.972 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:57:13.972 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ compare_image_snaps mirror image_22 snap 2024-09-14T08:57:13.972 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:13.972 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_22 2024-09-14T08:57:13.972 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:57:13.972 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local ret=0 2024-09-14T08:57:13.972 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local rmt_export=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_22.export 2024-09-14T08:57:13.972 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local loc_export=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_22.export 2024-09-14T08:57:13.972 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_22.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_22.export 2024-09-14T08:57:13.972 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror export image_22@snap /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_22.export 2024-09-14T08:57:14.178 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T08:57:14.184 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 -p mirror export image_22@snap /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_22.export 2024-09-14T08:57:14.478 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T08:57:14.485 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cmp /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_22.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_22.export 2024-09-14T08:57:14.540 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_22.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_22.export 2024-09-14T08:57:14.571 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:57:14.571 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T08:57:14.571 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_23 2024-09-14T08:57:14.571 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_snap cluster2 mirror image_23 snap 2024-09-14T08:57:14.571 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:57:14.571 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:14.571 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_23 2024-09-14T08:57:14.572 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:57:14.572 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror snap create image_23@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T08:57:15.307 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_started cluster1 mirror image_23 2024-09-14T08:57:15.308 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:57:15.308 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:15.308 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_23 2024-09-14T08:57:15.308 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_state cluster1 mirror image_23 started 2024-09-14T08:57:15.308 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:57:15.308 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:15.308 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_23 2024-09-14T08:57:15.308 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=started 2024-09-14T08:57:15.308 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:57:15.308 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:57:15.308 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:57:16.308 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_replay_state cluster1 mirror image_23 started 2024-09-14T08:57:16.309 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:57:16.309 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:16.309 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_23 2024-09-14T08:57:16.309 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=started 2024-09-14T08:57:16.309 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_result 2024-09-14T08:57:16.309 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=stopped 2024-09-14T08:57:16.310 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_23 2024-09-14T08:57:16.310 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ grep -i state 2024-09-14T08:57:16.310 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_instance=cluster1 2024-09-14T08:57:16.310 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:57:16.311 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T08:57:16.311 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance=cluster1 2024-09-14T08:57:16.311 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local loop_instance 2024-09-14T08:57:16.311 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:57:16.311 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sleep 0 2024-09-14T08:57:16.311 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T08:57:16.314 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ seq 0 3 2024-09-14T08:57:16.314 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:57:16.314 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_23 2024-09-14T08:57:16.314 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:0 2024-09-14T08:57:16.314 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:57:16.314 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:57:16.314 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T08:57:16.314 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:0 2024-09-14T08:57:16.314 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:57:16.315 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:57:16.315 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:57:16.315 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=0 2024-09-14T08:57:16.315 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T08:57:16.315 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:57:16.315 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:57:16.315 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=0 2024-09-14T08:57:16.315 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:57:16.315 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T08:57:16.315 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:0 2024-09-14T08:57:16.315 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:57:16.315 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:57:16.315 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T08:57:16.315 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:0 2024-09-14T08:57:16.315 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:57:16.316 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:57:16.316 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:57:16.316 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:57:16.316 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T08:57:16.316 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:57:16.316 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:57:16.316 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=0 2024-09-14T08:57:16.316 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=0 2024-09-14T08:57:16.316 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T08:57:16.335 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:57:16.335 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:57:16.335 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:57:16.336 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_23 2024-09-14T08:57:16.427 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_23'] 2024-09-14T08:57:16.427 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:57:16.428 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:57:16.430 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:57:16.430 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_23 2024-09-14T08:57:16.431 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:1 2024-09-14T08:57:16.431 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:57:16.431 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:57:16.431 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T08:57:16.431 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:1 2024-09-14T08:57:16.431 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:57:16.431 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:57:16.431 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:57:16.431 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=1 2024-09-14T08:57:16.431 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T08:57:16.431 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:57:16.431 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:57:16.431 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=1 2024-09-14T08:57:16.431 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:57:16.432 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T08:57:16.432 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:1 2024-09-14T08:57:16.432 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:57:16.432 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:57:16.432 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T08:57:16.432 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:1 2024-09-14T08:57:16.432 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:57:16.432 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:57:16.432 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:57:16.432 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:57:16.432 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T08:57:16.432 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:57:16.432 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:57:16.432 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=1 2024-09-14T08:57:16.432 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=1 2024-09-14T08:57:16.433 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T08:57:16.449 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:57:16.449 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:57:16.449 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:57:16.449 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_23 2024-09-14T08:57:16.537 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ return 0 2024-09-14T08:57:16.537 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T08:57:16.538 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ echo ' "state": "Replaying"' 2024-09-14T08:57:16.538 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep -i Replaying 2024-09-14T08:57:16.538 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: "state": "Replaying" 2024-09-14T08:57:16.539 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_state=started 2024-09-14T08:57:16.540 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test started = started 2024-09-14T08:57:16.540 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:57:16.540 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_23 2024-09-14T08:57:16.540 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:57:16.540 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:57:16.540 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:16.540 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_23 2024-09-14T08:57:16.540 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = journal ']' 2024-09-14T08:57:16.540 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = snapshot ']' 2024-09-14T08:57:16.540 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_23 2024-09-14T08:57:16.540 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:57:16.540 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:57:16.541 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:16.541 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_23 2024-09-14T08:57:16.541 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster2-mirror-image_23.status 2024-09-14T08:57:16.541 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster2-mirror-image_23.status 2024-09-14T08:57:16.541 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:57:16.541 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_23.status 2024-09-14T08:57:16.542 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster1-mirror-image_23.status 2024-09-14T08:57:16.542 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster1-mirror-image_23.status 2024-09-14T08:57:16.542 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:57:16.542 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_status_log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_23.status 2024-09-14T08:57:16.543 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ mirror_image_snapshot cluster2 mirror image_23 2024-09-14T08:57:16.543 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:57:16.543 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:16.543 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_23 2024-09-14T08:57:16.543 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_23 2024-09-14T08:57:17.324 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Snapshot ID: 130 2024-09-14T08:57:17.330 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_23 /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_23.status 2024-09-14T08:57:17.330 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:57:17.331 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:17.331 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_23 2024-09-14T08:57:17.331 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_23.status 2024-09-14T08:57:17.331 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_23 --format xml 2024-09-14T08:57:17.331 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:57:17.381 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-14T08:57:17.382 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snapshot_id=130 2024-09-14T08:57:17.382 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:57:17.383 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:57:17.383 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.2 2024-09-14T08:57:17.584 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_23 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_23.status 2024-09-14T08:57:17.584 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:57:17.584 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:17.584 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_23 2024-09-14T08:57:17.584 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_23.status 2024-09-14T08:57:17.584 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_23 --format xml 2024-09-14T08:57:17.584 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:57:17.634 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:57:17.635 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:57:17.636 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:57:17.636 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:57:17.636 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:57:17.636 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:57:17.636 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 130 = '' 2024-09-14T08:57:17.636 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:57:17.637 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.4 2024-09-14T08:57:18.038 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_23 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_23.status 2024-09-14T08:57:18.038 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:57:18.038 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:18.039 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_23 2024-09-14T08:57:18.039 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_23.status 2024-09-14T08:57:18.039 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_23 --format xml 2024-09-14T08:57:18.039 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:57:18.090 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:57:18.091 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:57:18.092 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:57:18.092 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:57:18.092 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:57:18.093 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:57:18.093 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 130 = '' 2024-09-14T08:57:18.093 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:57:18.093 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.8 2024-09-14T08:57:18.894 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_23 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_23.status 2024-09-14T08:57:18.894 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:57:18.894 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:18.894 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_23 2024-09-14T08:57:18.894 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_23.status 2024-09-14T08:57:18.894 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_23 --format xml 2024-09-14T08:57:18.894 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:57:18.941 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:57:18.943 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=130 2024-09-14T08:57:18.944 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 130 = 130 2024-09-14T08:57:18.944 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:57:18.944 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snap_present cluster1 mirror image_23 snap 2024-09-14T08:57:18.944 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:57:18.944 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:18.944 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_23 2024-09-14T08:57:18.944 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:57:18.944 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:57:18.944 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T08:57:18.944 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:57:19.255 INFO:tasks.ceph.cluster2.osd.2.smithi164.stderr:2024-09-14T08:57:19.255+0000 7fe32301f640 -1 reset not still connected to 0x55e770c156c0 2024-09-14T08:57:19.944 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 info mirror/image_23@snap 2024-09-14T08:57:19.989 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_23': 2024-09-14T08:57:19.989 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T08:57:19.989 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:57:19.989 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 2 2024-09-14T08:57:19.989 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 1296f65c03da 2024-09-14T08:57:19.989 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.1296f65c03da 2024-09-14T08:57:19.989 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:57:19.989 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering, non-primary 2024-09-14T08:57:19.989 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:57:19.989 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:57:19.989 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:48:45 2024 2024-09-14T08:57:19.989 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:48:45 2024 2024-09-14T08:57:19.990 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:48:45 2024 2024-09-14T08:57:19.990 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: protected: False 2024-09-14T08:57:19.990 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:57:19.990 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:57:19.990 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: b97e8af3-4a7b-444b-9f49-8151fd4bf536 2024-09-14T08:57:19.990 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: false 2024-09-14T08:57:19.992 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:57:19.993 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ compare_image_snaps mirror image_23 snap 2024-09-14T08:57:19.993 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:19.993 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_23 2024-09-14T08:57:19.993 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:57:19.993 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local ret=0 2024-09-14T08:57:19.993 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local rmt_export=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_23.export 2024-09-14T08:57:19.993 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local loc_export=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_23.export 2024-09-14T08:57:19.993 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_23.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_23.export 2024-09-14T08:57:19.993 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror export image_23@snap /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_23.export 2024-09-14T08:57:20.163 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.7876972170729237) > weight (0.25) 2024-09-14T08:57:20.163 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.48999230713273945) > weight (0.25) 2024-09-14T08:57:20.163 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.9471566852843922) > weight (0.25) 2024-09-14T08:57:20.163 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.3 with skip (0.5196785938064877) > weight (0.25) 2024-09-14T08:57:20.163 INFO:tasks.rbd_mirror_thrash:waiting for 85 secs before thrashing 2024-09-14T08:57:20.208 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T08:57:20.215 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 -p mirror export image_23@snap /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_23.export 2024-09-14T08:57:20.470 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T08:57:20.477 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cmp /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_23.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_23.export 2024-09-14T08:57:20.532 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_23.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_23.export 2024-09-14T08:57:20.564 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:57:20.564 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T08:57:20.564 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_24 2024-09-14T08:57:20.564 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_snap cluster2 mirror image_24 snap 2024-09-14T08:57:20.564 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:57:20.564 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:20.565 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_24 2024-09-14T08:57:20.565 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:57:20.565 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror snap create image_24@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T08:57:21.318 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_started cluster1 mirror image_24 2024-09-14T08:57:21.318 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:57:21.318 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:21.318 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_24 2024-09-14T08:57:21.318 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_state cluster1 mirror image_24 started 2024-09-14T08:57:21.318 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:57:21.319 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:21.319 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_24 2024-09-14T08:57:21.319 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=started 2024-09-14T08:57:21.319 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:57:21.319 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:57:21.319 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:57:22.320 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_replay_state cluster1 mirror image_24 started 2024-09-14T08:57:22.320 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:57:22.320 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:22.320 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_24 2024-09-14T08:57:22.320 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=started 2024-09-14T08:57:22.320 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_result 2024-09-14T08:57:22.320 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=stopped 2024-09-14T08:57:22.320 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_24 2024-09-14T08:57:22.321 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_instance=cluster1 2024-09-14T08:57:22.321 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:57:22.321 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ grep -i state 2024-09-14T08:57:22.321 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T08:57:22.321 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance=cluster1 2024-09-14T08:57:22.321 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local loop_instance 2024-09-14T08:57:22.321 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:57:22.321 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sleep 0 2024-09-14T08:57:22.323 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T08:57:22.324 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ seq 0 3 2024-09-14T08:57:22.324 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:57:22.324 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_24 2024-09-14T08:57:22.324 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:0 2024-09-14T08:57:22.324 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:57:22.324 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:57:22.324 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T08:57:22.324 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:0 2024-09-14T08:57:22.324 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:57:22.324 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:57:22.324 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:57:22.324 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=0 2024-09-14T08:57:22.325 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T08:57:22.325 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:57:22.325 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:57:22.325 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=0 2024-09-14T08:57:22.325 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:57:22.325 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T08:57:22.325 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:0 2024-09-14T08:57:22.325 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:57:22.325 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:57:22.325 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T08:57:22.325 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:0 2024-09-14T08:57:22.325 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:57:22.325 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:57:22.325 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:57:22.326 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:57:22.326 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T08:57:22.326 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:57:22.326 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:57:22.326 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=0 2024-09-14T08:57:22.326 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=0 2024-09-14T08:57:22.326 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T08:57:22.344 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:57:22.344 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:57:22.344 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:57:22.345 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_24 2024-09-14T08:57:22.435 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ return 0 2024-09-14T08:57:22.436 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T08:57:22.436 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ echo ' "state": "Replaying"' 2024-09-14T08:57:22.436 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep -i Replaying 2024-09-14T08:57:22.436 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: "state": "Replaying" 2024-09-14T08:57:22.438 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_state=started 2024-09-14T08:57:22.438 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test started = started 2024-09-14T08:57:22.438 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:57:22.438 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_24 2024-09-14T08:57:22.438 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:57:22.438 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:57:22.438 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:22.438 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_24 2024-09-14T08:57:22.439 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = journal ']' 2024-09-14T08:57:22.439 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = snapshot ']' 2024-09-14T08:57:22.439 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_24 2024-09-14T08:57:22.439 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:57:22.439 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:57:22.439 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:22.439 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_24 2024-09-14T08:57:22.439 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster2-mirror-image_24.status 2024-09-14T08:57:22.439 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster2-mirror-image_24.status 2024-09-14T08:57:22.439 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:57:22.440 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_24.status 2024-09-14T08:57:22.440 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster1-mirror-image_24.status 2024-09-14T08:57:22.441 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster1-mirror-image_24.status 2024-09-14T08:57:22.441 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:57:22.441 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_status_log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_24.status 2024-09-14T08:57:22.441 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ mirror_image_snapshot cluster2 mirror image_24 2024-09-14T08:57:22.441 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:57:22.441 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:22.441 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_24 2024-09-14T08:57:22.441 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_24 2024-09-14T08:57:23.330 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Snapshot ID: 132 2024-09-14T08:57:23.336 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_24 /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_24.status 2024-09-14T08:57:23.336 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:57:23.336 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:23.336 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_24 2024-09-14T08:57:23.336 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_24.status 2024-09-14T08:57:23.337 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_24 --format xml 2024-09-14T08:57:23.337 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:57:23.384 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-14T08:57:23.386 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snapshot_id=132 2024-09-14T08:57:23.386 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:57:23.386 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:57:23.386 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.2 2024-09-14T08:57:23.587 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_24 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_24.status 2024-09-14T08:57:23.587 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:57:23.587 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:23.587 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_24 2024-09-14T08:57:23.587 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_24.status 2024-09-14T08:57:23.587 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_24 --format xml 2024-09-14T08:57:23.587 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:57:23.635 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:57:23.635 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:57:23.636 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:57:23.637 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:57:23.637 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:57:23.637 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:57:23.637 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 132 = '' 2024-09-14T08:57:23.637 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:57:23.637 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.4 2024-09-14T08:57:24.038 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_24 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_24.status 2024-09-14T08:57:24.039 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:57:24.039 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:24.039 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_24 2024-09-14T08:57:24.039 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_24.status 2024-09-14T08:57:24.039 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_24 --format xml 2024-09-14T08:57:24.039 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:57:24.088 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:57:24.088 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:57:24.089 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:57:24.090 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:57:24.090 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:57:24.090 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:57:24.090 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 132 = '' 2024-09-14T08:57:24.090 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:57:24.091 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.8 2024-09-14T08:57:24.892 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_24 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_24.status 2024-09-14T08:57:24.892 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:57:24.892 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:24.892 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_24 2024-09-14T08:57:24.892 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_24.status 2024-09-14T08:57:24.892 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_24 --format xml 2024-09-14T08:57:24.892 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:57:24.940 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:57:24.942 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=132 2024-09-14T08:57:24.942 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 132 = 132 2024-09-14T08:57:24.942 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:57:24.942 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snap_present cluster1 mirror image_24 snap 2024-09-14T08:57:24.942 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:57:24.942 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:24.942 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_24 2024-09-14T08:57:24.942 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:57:24.942 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:57:24.942 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T08:57:24.943 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:57:25.943 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 info mirror/image_24@snap 2024-09-14T08:57:25.990 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_24': 2024-09-14T08:57:25.991 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T08:57:25.991 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:57:25.991 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 2 2024-09-14T08:57:25.991 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 1296acecd8f 2024-09-14T08:57:25.991 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.1296acecd8f 2024-09-14T08:57:25.991 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:57:25.991 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering, non-primary 2024-09-14T08:57:25.991 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:57:25.991 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:57:25.991 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:48:57 2024 2024-09-14T08:57:25.992 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:48:57 2024 2024-09-14T08:57:25.992 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:48:57 2024 2024-09-14T08:57:25.992 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: protected: False 2024-09-14T08:57:25.992 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:57:25.992 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:57:25.992 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: 24c48f34-7f6e-4594-86fc-0a765052746d 2024-09-14T08:57:25.992 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: false 2024-09-14T08:57:25.997 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:57:25.997 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ compare_image_snaps mirror image_24 snap 2024-09-14T08:57:25.997 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:25.998 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_24 2024-09-14T08:57:25.998 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:57:25.998 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local ret=0 2024-09-14T08:57:25.998 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local rmt_export=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_24.export 2024-09-14T08:57:25.998 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local loc_export=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_24.export 2024-09-14T08:57:25.998 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_24.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_24.export 2024-09-14T08:57:25.998 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror export image_24@snap /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_24.export 2024-09-14T08:57:26.185 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T08:57:26.193 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 -p mirror export image_24@snap /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_24.export 2024-09-14T08:57:26.404 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T08:57:26.409 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cmp /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_24.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_24.export 2024-09-14T08:57:26.463 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_24.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_24.export 2024-09-14T08:57:26.495 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:57:26.495 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T08:57:26.495 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_25 2024-09-14T08:57:26.495 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_snap cluster2 mirror image_25 snap 2024-09-14T08:57:26.495 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:57:26.495 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:26.495 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_25 2024-09-14T08:57:26.495 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:57:26.495 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror snap create image_25@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T08:57:27.328 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_started cluster1 mirror image_25 2024-09-14T08:57:27.328 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:57:27.328 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:27.328 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_25 2024-09-14T08:57:27.328 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_state cluster1 mirror image_25 started 2024-09-14T08:57:27.329 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:57:27.329 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:27.329 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_25 2024-09-14T08:57:27.329 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=started 2024-09-14T08:57:27.329 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:57:27.329 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:57:27.329 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:57:28.329 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_replay_state cluster1 mirror image_25 started 2024-09-14T08:57:28.329 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:57:28.329 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:28.330 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_25 2024-09-14T08:57:28.330 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=started 2024-09-14T08:57:28.330 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_result 2024-09-14T08:57:28.330 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=stopped 2024-09-14T08:57:28.331 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_25 2024-09-14T08:57:28.331 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_instance=cluster1 2024-09-14T08:57:28.331 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ grep -i state 2024-09-14T08:57:28.331 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:57:28.331 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T08:57:28.331 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance=cluster1 2024-09-14T08:57:28.331 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local loop_instance 2024-09-14T08:57:28.331 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:57:28.331 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sleep 0 2024-09-14T08:57:28.331 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T08:57:28.334 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ seq 0 3 2024-09-14T08:57:28.334 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:57:28.334 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_25 2024-09-14T08:57:28.334 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:0 2024-09-14T08:57:28.334 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:57:28.334 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:57:28.334 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T08:57:28.334 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:0 2024-09-14T08:57:28.335 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:57:28.335 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:57:28.335 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:57:28.335 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=0 2024-09-14T08:57:28.335 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T08:57:28.335 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:57:28.335 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:57:28.335 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=0 2024-09-14T08:57:28.335 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:57:28.335 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T08:57:28.335 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:0 2024-09-14T08:57:28.335 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:57:28.335 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:57:28.335 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T08:57:28.336 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:0 2024-09-14T08:57:28.336 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:57:28.336 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:57:28.336 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:57:28.336 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:57:28.336 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T08:57:28.336 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:57:28.336 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:57:28.336 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=0 2024-09-14T08:57:28.336 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=0 2024-09-14T08:57:28.336 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T08:57:28.354 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:57:28.355 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:57:28.355 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:57:28.355 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_25 2024-09-14T08:57:28.440 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_25'] 2024-09-14T08:57:28.440 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:57:28.440 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:57:28.443 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:57:28.443 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_25 2024-09-14T08:57:28.443 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:1 2024-09-14T08:57:28.443 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:57:28.443 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:57:28.443 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T08:57:28.443 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:1 2024-09-14T08:57:28.443 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:57:28.443 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:57:28.443 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:57:28.444 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=1 2024-09-14T08:57:28.444 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T08:57:28.444 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:57:28.444 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:57:28.444 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=1 2024-09-14T08:57:28.444 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:57:28.444 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T08:57:28.444 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:1 2024-09-14T08:57:28.444 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:57:28.444 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:57:28.444 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T08:57:28.444 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:1 2024-09-14T08:57:28.444 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:57:28.444 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:57:28.444 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:57:28.445 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:57:28.445 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T08:57:28.445 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:57:28.445 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:57:28.445 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=1 2024-09-14T08:57:28.445 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=1 2024-09-14T08:57:28.445 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T08:57:28.463 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:57:28.463 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:57:28.463 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:57:28.463 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_25 2024-09-14T08:57:28.544 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_25'] 2024-09-14T08:57:28.544 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:57:28.544 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:57:28.547 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:57:28.547 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_25 2024-09-14T08:57:28.548 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:2 2024-09-14T08:57:28.548 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:57:28.548 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:57:28.548 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-14T08:57:28.548 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:2 2024-09-14T08:57:28.548 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:57:28.548 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:57:28.548 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:57:28.548 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=2 2024-09-14T08:57:28.548 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T08:57:28.548 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:57:28.548 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:57:28.548 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=2 2024-09-14T08:57:28.549 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:57:28.549 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-14T08:57:28.549 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:2 2024-09-14T08:57:28.549 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:57:28.549 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:57:28.549 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T08:57:28.549 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:2 2024-09-14T08:57:28.549 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:57:28.549 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:57:28.549 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:57:28.550 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:57:28.550 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-14T08:57:28.550 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:57:28.550 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:57:28.550 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=2 2024-09-14T08:57:28.550 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=2 2024-09-14T08:57:28.550 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T08:57:28.566 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:57:28.566 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:57:28.566 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:57:28.566 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_25 2024-09-14T08:57:28.655 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ return 0 2024-09-14T08:57:28.656 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T08:57:28.656 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ echo ' "state": "Replaying"' 2024-09-14T08:57:28.656 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep -i Replaying 2024-09-14T08:57:28.657 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: "state": "Replaying" 2024-09-14T08:57:28.658 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_state=started 2024-09-14T08:57:28.658 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test started = started 2024-09-14T08:57:28.658 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:57:28.658 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_25 2024-09-14T08:57:28.659 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:57:28.659 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:57:28.659 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:28.659 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_25 2024-09-14T08:57:28.659 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = journal ']' 2024-09-14T08:57:28.659 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = snapshot ']' 2024-09-14T08:57:28.659 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_25 2024-09-14T08:57:28.659 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:57:28.660 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:57:28.660 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:28.660 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_25 2024-09-14T08:57:28.660 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster2-mirror-image_25.status 2024-09-14T08:57:28.660 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster2-mirror-image_25.status 2024-09-14T08:57:28.660 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:57:28.661 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_25.status 2024-09-14T08:57:28.661 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster1-mirror-image_25.status 2024-09-14T08:57:28.661 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster1-mirror-image_25.status 2024-09-14T08:57:28.661 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:57:28.661 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_status_log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_25.status 2024-09-14T08:57:28.662 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ mirror_image_snapshot cluster2 mirror image_25 2024-09-14T08:57:28.662 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:57:28.662 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:28.662 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_25 2024-09-14T08:57:28.662 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_25 2024-09-14T08:57:29.337 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Snapshot ID: 134 2024-09-14T08:57:29.343 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_25 /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_25.status 2024-09-14T08:57:29.344 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:57:29.344 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:29.344 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_25 2024-09-14T08:57:29.344 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_25.status 2024-09-14T08:57:29.344 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_25 --format xml 2024-09-14T08:57:29.344 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:57:29.392 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-14T08:57:29.394 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snapshot_id=134 2024-09-14T08:57:29.394 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:57:29.394 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:57:29.394 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.2 2024-09-14T08:57:29.595 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_25 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_25.status 2024-09-14T08:57:29.596 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:57:29.596 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:29.596 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_25 2024-09-14T08:57:29.596 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_25.status 2024-09-14T08:57:29.596 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_25 --format xml 2024-09-14T08:57:29.596 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:57:29.645 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:57:29.646 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:57:29.647 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:57:29.647 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:57:29.647 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:57:29.648 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:57:29.648 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 134 = '' 2024-09-14T08:57:29.648 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:57:29.648 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.4 2024-09-14T08:57:30.049 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_25 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_25.status 2024-09-14T08:57:30.049 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:57:30.049 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:30.049 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_25 2024-09-14T08:57:30.050 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_25.status 2024-09-14T08:57:30.050 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_25 --format xml 2024-09-14T08:57:30.050 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:57:30.098 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:57:30.098 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:57:30.100 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:57:30.100 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:57:30.100 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:57:30.100 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:57:30.100 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 134 = '' 2024-09-14T08:57:30.100 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:57:30.101 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.8 2024-09-14T08:57:30.904 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_25 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_25.status 2024-09-14T08:57:30.905 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:57:30.905 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:30.905 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_25 2024-09-14T08:57:30.905 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_25.status 2024-09-14T08:57:30.905 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_25 --format xml 2024-09-14T08:57:30.905 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:57:30.950 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:57:30.952 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=134 2024-09-14T08:57:30.952 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 134 = 134 2024-09-14T08:57:30.952 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:57:30.952 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snap_present cluster1 mirror image_25 snap 2024-09-14T08:57:30.952 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:57:30.952 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:30.952 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_25 2024-09-14T08:57:30.952 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:57:30.952 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:57:30.952 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T08:57:30.952 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:57:31.953 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 info mirror/image_25@snap 2024-09-14T08:57:31.997 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_25': 2024-09-14T08:57:31.997 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T08:57:31.997 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:57:31.997 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 2 2024-09-14T08:57:31.998 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 10bda9c3d78f 2024-09-14T08:57:31.998 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.10bda9c3d78f 2024-09-14T08:57:31.998 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:57:31.998 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering, non-primary 2024-09-14T08:57:31.998 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:57:31.998 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:57:31.998 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:49:07 2024 2024-09-14T08:57:31.998 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:49:07 2024 2024-09-14T08:57:31.998 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:49:07 2024 2024-09-14T08:57:31.998 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: protected: False 2024-09-14T08:57:31.998 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:57:31.998 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:57:31.998 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: 3d365e96-a01e-4587-97b2-7c3efff31e05 2024-09-14T08:57:31.998 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: false 2024-09-14T08:57:32.001 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:57:32.001 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ compare_image_snaps mirror image_25 snap 2024-09-14T08:57:32.001 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:32.001 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_25 2024-09-14T08:57:32.001 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:57:32.001 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local ret=0 2024-09-14T08:57:32.001 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local rmt_export=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_25.export 2024-09-14T08:57:32.001 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local loc_export=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_25.export 2024-09-14T08:57:32.001 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_25.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_25.export 2024-09-14T08:57:32.002 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror export image_25@snap /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_25.export 2024-09-14T08:57:32.197 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T08:57:32.205 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 -p mirror export image_25@snap /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_25.export 2024-09-14T08:57:32.450 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T08:57:32.458 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cmp /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_25.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_25.export 2024-09-14T08:57:32.512 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_25.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_25.export 2024-09-14T08:57:32.544 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:57:32.544 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T08:57:32.544 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_26 2024-09-14T08:57:32.544 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_snap cluster2 mirror image_26 snap 2024-09-14T08:57:32.544 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:57:32.544 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:32.544 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_26 2024-09-14T08:57:32.545 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:57:32.545 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror snap create image_26@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T08:57:33.345 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_started cluster1 mirror image_26 2024-09-14T08:57:33.346 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:57:33.346 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:33.346 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_26 2024-09-14T08:57:33.346 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_state cluster1 mirror image_26 started 2024-09-14T08:57:33.346 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:57:33.346 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:33.346 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_26 2024-09-14T08:57:33.346 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=started 2024-09-14T08:57:33.346 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:57:33.346 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:57:33.346 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:57:34.346 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_replay_state cluster1 mirror image_26 started 2024-09-14T08:57:34.347 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:57:34.347 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:34.347 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_26 2024-09-14T08:57:34.347 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=started 2024-09-14T08:57:34.347 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_result 2024-09-14T08:57:34.347 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=stopped 2024-09-14T08:57:34.348 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_26 2024-09-14T08:57:34.348 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_instance=cluster1 2024-09-14T08:57:34.349 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:57:34.349 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ grep -i state 2024-09-14T08:57:34.349 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T08:57:34.349 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance=cluster1 2024-09-14T08:57:34.349 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local loop_instance 2024-09-14T08:57:34.349 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:57:34.349 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sleep 0 2024-09-14T08:57:34.349 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T08:57:34.349 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ seq 0 3 2024-09-14T08:57:34.352 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:57:34.352 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_26 2024-09-14T08:57:34.352 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:0 2024-09-14T08:57:34.352 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:57:34.352 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:57:34.352 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T08:57:34.352 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:0 2024-09-14T08:57:34.352 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:57:34.353 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:57:34.353 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:57:34.353 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=0 2024-09-14T08:57:34.353 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T08:57:34.353 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:57:34.353 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:57:34.353 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=0 2024-09-14T08:57:34.353 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:57:34.353 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T08:57:34.353 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:0 2024-09-14T08:57:34.353 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:57:34.353 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:57:34.353 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T08:57:34.353 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:0 2024-09-14T08:57:34.353 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:57:34.354 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:57:34.354 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:57:34.354 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:57:34.354 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T08:57:34.354 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:57:34.354 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:57:34.354 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=0 2024-09-14T08:57:34.354 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=0 2024-09-14T08:57:34.354 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T08:57:34.373 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:57:34.373 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:57:34.373 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:57:34.373 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_26 2024-09-14T08:57:34.469 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ return 0 2024-09-14T08:57:34.470 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T08:57:34.470 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ echo ' "state": "Replaying"' 2024-09-14T08:57:34.470 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep -i Replaying 2024-09-14T08:57:34.471 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: "state": "Replaying" 2024-09-14T08:57:34.472 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_state=started 2024-09-14T08:57:34.472 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test started = started 2024-09-14T08:57:34.472 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:57:34.473 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_26 2024-09-14T08:57:34.473 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:57:34.473 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:57:34.473 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:34.473 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_26 2024-09-14T08:57:34.473 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = journal ']' 2024-09-14T08:57:34.473 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = snapshot ']' 2024-09-14T08:57:34.473 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_26 2024-09-14T08:57:34.473 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:57:34.473 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:57:34.473 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:34.473 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_26 2024-09-14T08:57:34.473 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster2-mirror-image_26.status 2024-09-14T08:57:34.473 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster2-mirror-image_26.status 2024-09-14T08:57:34.473 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:57:34.474 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_26.status 2024-09-14T08:57:34.474 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster1-mirror-image_26.status 2024-09-14T08:57:34.474 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster1-mirror-image_26.status 2024-09-14T08:57:34.474 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:57:34.475 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_status_log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_26.status 2024-09-14T08:57:34.475 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ mirror_image_snapshot cluster2 mirror image_26 2024-09-14T08:57:34.475 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:57:34.476 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:34.476 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_26 2024-09-14T08:57:34.476 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_26 2024-09-14T08:57:35.352 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Snapshot ID: 136 2024-09-14T08:57:35.360 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_26 /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_26.status 2024-09-14T08:57:35.360 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:57:35.360 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:35.360 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_26 2024-09-14T08:57:35.360 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_26.status 2024-09-14T08:57:35.360 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_26 --format xml 2024-09-14T08:57:35.360 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:57:35.409 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-14T08:57:35.411 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snapshot_id=136 2024-09-14T08:57:35.411 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:57:35.411 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:57:35.411 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.2 2024-09-14T08:57:35.612 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_26 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_26.status 2024-09-14T08:57:35.612 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:57:35.612 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:35.612 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_26 2024-09-14T08:57:35.612 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_26.status 2024-09-14T08:57:35.612 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_26 --format xml 2024-09-14T08:57:35.613 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:57:35.665 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:57:35.665 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:57:35.667 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:57:35.667 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:57:35.667 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:57:35.667 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:57:35.668 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 136 = '' 2024-09-14T08:57:35.668 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:57:35.668 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.4 2024-09-14T08:57:36.069 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_26 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_26.status 2024-09-14T08:57:36.069 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:57:36.069 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:36.069 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_26 2024-09-14T08:57:36.069 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_26.status 2024-09-14T08:57:36.069 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_26 --format xml 2024-09-14T08:57:36.069 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:57:36.123 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:57:36.124 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:57:36.125 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:57:36.125 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:57:36.125 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:57:36.126 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:57:36.126 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 136 = '' 2024-09-14T08:57:36.126 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:57:36.126 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.8 2024-09-14T08:57:36.927 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_26 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_26.status 2024-09-14T08:57:36.928 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:57:36.928 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:36.928 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_26 2024-09-14T08:57:36.928 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_26.status 2024-09-14T08:57:36.928 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_26 --format xml 2024-09-14T08:57:36.928 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:57:36.976 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:57:36.978 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=136 2024-09-14T08:57:36.978 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 136 = 136 2024-09-14T08:57:36.978 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:57:36.978 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snap_present cluster1 mirror image_26 snap 2024-09-14T08:57:36.978 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:57:36.978 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:36.978 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_26 2024-09-14T08:57:36.978 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:57:36.979 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:57:36.979 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T08:57:36.979 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:57:37.979 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 info mirror/image_26@snap 2024-09-14T08:57:38.026 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_26': 2024-09-14T08:57:38.026 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T08:57:38.026 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:57:38.026 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 2 2024-09-14T08:57:38.026 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 118911e4f814 2024-09-14T08:57:38.026 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.118911e4f814 2024-09-14T08:57:38.026 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:57:38.027 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering, non-primary 2024-09-14T08:57:38.027 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:57:38.027 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:57:38.027 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:49:14 2024 2024-09-14T08:57:38.027 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:49:14 2024 2024-09-14T08:57:38.027 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:49:14 2024 2024-09-14T08:57:38.027 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: protected: False 2024-09-14T08:57:38.027 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:57:38.027 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:57:38.027 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: 4ec19e01-86f0-46e8-9aed-ae17d2da69c7 2024-09-14T08:57:38.027 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: false 2024-09-14T08:57:38.030 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:57:38.030 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ compare_image_snaps mirror image_26 snap 2024-09-14T08:57:38.030 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:38.030 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_26 2024-09-14T08:57:38.030 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:57:38.030 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local ret=0 2024-09-14T08:57:38.030 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local rmt_export=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_26.export 2024-09-14T08:57:38.030 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local loc_export=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_26.export 2024-09-14T08:57:38.030 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_26.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_26.export 2024-09-14T08:57:38.031 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror export image_26@snap /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_26.export 2024-09-14T08:57:38.220 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T08:57:38.228 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 -p mirror export image_26@snap /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_26.export 2024-09-14T08:57:38.453 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T08:57:38.462 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cmp /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_26.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_26.export 2024-09-14T08:57:38.519 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_26.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_26.export 2024-09-14T08:57:38.547 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:57:38.547 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T08:57:38.547 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_27 2024-09-14T08:57:38.548 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_snap cluster2 mirror image_27 snap 2024-09-14T08:57:38.548 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:57:38.548 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:38.548 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_27 2024-09-14T08:57:38.548 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:57:38.548 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror snap create image_27@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T08:57:39.346 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_started cluster1 mirror image_27 2024-09-14T08:57:39.346 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:57:39.346 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:39.346 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_27 2024-09-14T08:57:39.346 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_state cluster1 mirror image_27 started 2024-09-14T08:57:39.346 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:57:39.346 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:39.346 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_27 2024-09-14T08:57:39.346 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=started 2024-09-14T08:57:39.347 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:57:39.347 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:57:39.347 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:57:40.347 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_replay_state cluster1 mirror image_27 started 2024-09-14T08:57:40.347 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:57:40.347 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:40.347 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_27 2024-09-14T08:57:40.347 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=started 2024-09-14T08:57:40.347 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_result 2024-09-14T08:57:40.347 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=stopped 2024-09-14T08:57:40.348 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_27 2024-09-14T08:57:40.349 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_instance=cluster1 2024-09-14T08:57:40.349 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:57:40.349 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T08:57:40.349 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance=cluster1 2024-09-14T08:57:40.349 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ grep -i state 2024-09-14T08:57:40.349 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local loop_instance 2024-09-14T08:57:40.349 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:57:40.349 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sleep 0 2024-09-14T08:57:40.349 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T08:57:40.352 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ seq 0 3 2024-09-14T08:57:40.352 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:57:40.352 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_27 2024-09-14T08:57:40.352 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:0 2024-09-14T08:57:40.352 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:57:40.352 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:57:40.352 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T08:57:40.352 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:0 2024-09-14T08:57:40.352 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:57:40.352 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:57:40.353 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:57:40.353 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=0 2024-09-14T08:57:40.353 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T08:57:40.353 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:57:40.353 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:57:40.353 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=0 2024-09-14T08:57:40.353 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:57:40.353 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T08:57:40.353 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:0 2024-09-14T08:57:40.353 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:57:40.353 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:57:40.353 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T08:57:40.353 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:0 2024-09-14T08:57:40.354 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:57:40.354 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:57:40.354 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:57:40.354 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:57:40.354 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T08:57:40.354 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:57:40.354 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:57:40.354 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=0 2024-09-14T08:57:40.354 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=0 2024-09-14T08:57:40.354 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T08:57:40.373 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:57:40.374 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:57:40.374 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:57:40.374 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_27 2024-09-14T08:57:40.462 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_27'] 2024-09-14T08:57:40.462 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:57:40.462 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:57:40.465 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:57:40.465 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_27 2024-09-14T08:57:40.466 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:1 2024-09-14T08:57:40.466 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:57:40.466 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:57:40.466 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T08:57:40.466 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:1 2024-09-14T08:57:40.466 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:57:40.466 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:57:40.466 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:57:40.466 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=1 2024-09-14T08:57:40.466 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T08:57:40.466 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:57:40.466 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:57:40.466 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=1 2024-09-14T08:57:40.467 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:57:40.467 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T08:57:40.467 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:1 2024-09-14T08:57:40.467 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:57:40.467 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:57:40.467 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T08:57:40.467 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:1 2024-09-14T08:57:40.467 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:57:40.467 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:57:40.467 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:57:40.467 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:57:40.467 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T08:57:40.467 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:57:40.467 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:57:40.467 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=1 2024-09-14T08:57:40.468 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=1 2024-09-14T08:57:40.468 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T08:57:40.484 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:57:40.484 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:57:40.484 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:57:40.484 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_27 2024-09-14T08:57:40.567 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_27'] 2024-09-14T08:57:40.567 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:57:40.567 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:57:40.570 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:57:40.570 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_27 2024-09-14T08:57:40.570 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:2 2024-09-14T08:57:40.570 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:57:40.570 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:57:40.570 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-14T08:57:40.570 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:2 2024-09-14T08:57:40.570 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:57:40.570 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:57:40.570 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:57:40.571 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=2 2024-09-14T08:57:40.571 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T08:57:40.571 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:57:40.571 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:57:40.571 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=2 2024-09-14T08:57:40.571 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:57:40.571 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-14T08:57:40.571 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:2 2024-09-14T08:57:40.571 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:57:40.571 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:57:40.571 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T08:57:40.571 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:2 2024-09-14T08:57:40.571 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:57:40.571 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:57:40.572 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:57:40.572 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:57:40.572 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-14T08:57:40.572 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:57:40.572 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:57:40.572 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=2 2024-09-14T08:57:40.572 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=2 2024-09-14T08:57:40.572 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T08:57:40.588 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:57:40.588 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:57:40.588 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:57:40.589 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_27 2024-09-14T08:57:40.670 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_27'] 2024-09-14T08:57:40.670 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:57:40.670 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:57:40.673 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:57:40.673 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_27 2024-09-14T08:57:40.673 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:3 2024-09-14T08:57:40.673 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:57:40.673 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:57:40.673 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:3 cluster instance 2024-09-14T08:57:40.673 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:3 2024-09-14T08:57:40.673 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:57:40.673 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:57:40.673 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:57:40.674 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=3 2024-09-14T08:57:40.674 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-14T08:57:40.674 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:57:40.674 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:57:40.674 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=3 2024-09-14T08:57:40.674 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=3 2024-09-14T08:57:40.674 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:3 cluster1 2024-09-14T08:57:40.674 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:3 2024-09-14T08:57:40.674 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:57:40.674 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:57:40.674 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-14T08:57:40.674 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:3 2024-09-14T08:57:40.674 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:57:40.674 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:57:40.675 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:57:40.675 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=3 2024-09-14T08:57:40.675 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 3 = cluster1:3 ']' 2024-09-14T08:57:40.675 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:57:40.675 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:57:40.675 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=3 2024-09-14T08:57:40.675 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=3 2024-09-14T08:57:40.675 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-14T08:57:40.692 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T08:57:40.692 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T08:57:40.692 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T08:57:40.692 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_27 2024-09-14T08:57:40.782 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ return 0 2024-09-14T08:57:40.783 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T08:57:40.783 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ echo ' "state": "Replaying"' 2024-09-14T08:57:40.783 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep -i Replaying 2024-09-14T08:57:40.784 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: "state": "Replaying" 2024-09-14T08:57:40.785 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_state=started 2024-09-14T08:57:40.785 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test started = started 2024-09-14T08:57:40.785 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:57:40.785 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_27 2024-09-14T08:57:40.785 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:57:40.786 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:57:40.786 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:40.786 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_27 2024-09-14T08:57:40.786 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = journal ']' 2024-09-14T08:57:40.786 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = snapshot ']' 2024-09-14T08:57:40.786 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_27 2024-09-14T08:57:40.786 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:57:40.787 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:57:40.787 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:40.787 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_27 2024-09-14T08:57:40.787 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster2-mirror-image_27.status 2024-09-14T08:57:40.787 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster2-mirror-image_27.status 2024-09-14T08:57:40.787 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:57:40.789 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_27.status 2024-09-14T08:57:40.789 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster1-mirror-image_27.status 2024-09-14T08:57:40.789 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster1-mirror-image_27.status 2024-09-14T08:57:40.789 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:57:40.789 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_status_log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_27.status 2024-09-14T08:57:40.789 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ mirror_image_snapshot cluster2 mirror image_27 2024-09-14T08:57:40.789 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:57:40.789 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:40.789 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_27 2024-09-14T08:57:40.789 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_27 2024-09-14T08:57:41.354 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Snapshot ID: 138 2024-09-14T08:57:41.360 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_27 /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_27.status 2024-09-14T08:57:41.360 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:57:41.361 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:41.361 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_27 2024-09-14T08:57:41.361 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_27.status 2024-09-14T08:57:41.361 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_27 --format xml 2024-09-14T08:57:41.361 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:57:41.408 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-14T08:57:41.411 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snapshot_id=138 2024-09-14T08:57:41.411 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:57:41.411 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:57:41.411 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.2 2024-09-14T08:57:41.612 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_27 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_27.status 2024-09-14T08:57:41.612 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:57:41.612 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:41.612 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_27 2024-09-14T08:57:41.612 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_27.status 2024-09-14T08:57:41.612 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_27 --format xml 2024-09-14T08:57:41.612 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:57:41.660 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:57:41.661 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:57:41.662 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:57:41.662 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:57:41.662 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:57:41.662 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:57:41.662 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 138 = '' 2024-09-14T08:57:41.663 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:57:41.663 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.4 2024-09-14T08:57:42.064 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_27 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_27.status 2024-09-14T08:57:42.065 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:57:42.065 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:42.065 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_27 2024-09-14T08:57:42.065 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_27.status 2024-09-14T08:57:42.065 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_27 --format xml 2024-09-14T08:57:42.066 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:57:42.112 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:57:42.112 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:57:42.113 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:57:42.114 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:57:42.114 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:57:42.114 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:57:42.114 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 138 = '' 2024-09-14T08:57:42.114 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:57:42.115 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.8 2024-09-14T08:57:42.915 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_27 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_27.status 2024-09-14T08:57:42.916 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:57:42.916 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:42.916 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_27 2024-09-14T08:57:42.916 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_27.status 2024-09-14T08:57:42.916 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_27 --format xml 2024-09-14T08:57:42.916 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:57:42.964 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:57:42.965 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=138 2024-09-14T08:57:42.966 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 138 = 138 2024-09-14T08:57:42.966 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:57:42.966 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snap_present cluster1 mirror image_27 snap 2024-09-14T08:57:42.966 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:57:42.966 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:42.966 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_27 2024-09-14T08:57:42.966 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:57:42.966 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:57:42.966 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T08:57:42.966 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:57:43.966 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 info mirror/image_27@snap 2024-09-14T08:57:44.013 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_27': 2024-09-14T08:57:44.013 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T08:57:44.013 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:57:44.013 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 2 2024-09-14T08:57:44.013 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 127e38d16296 2024-09-14T08:57:44.013 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.127e38d16296 2024-09-14T08:57:44.013 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:57:44.013 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering, non-primary 2024-09-14T08:57:44.013 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:57:44.013 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:57:44.013 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:49:21 2024 2024-09-14T08:57:44.014 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:49:21 2024 2024-09-14T08:57:44.014 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:49:21 2024 2024-09-14T08:57:44.014 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: protected: False 2024-09-14T08:57:44.014 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:57:44.014 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:57:44.014 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: a44041c2-b983-4eef-b3f2-aec32b4ab580 2024-09-14T08:57:44.014 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: false 2024-09-14T08:57:44.017 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:57:44.017 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ compare_image_snaps mirror image_27 snap 2024-09-14T08:57:44.017 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:44.017 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_27 2024-09-14T08:57:44.017 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:57:44.017 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local ret=0 2024-09-14T08:57:44.017 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local rmt_export=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_27.export 2024-09-14T08:57:44.017 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local loc_export=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_27.export 2024-09-14T08:57:44.017 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_27.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_27.export 2024-09-14T08:57:44.018 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror export image_27@snap /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_27.export 2024-09-14T08:57:44.215 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T08:57:44.223 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 -p mirror export image_27@snap /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_27.export 2024-09-14T08:57:44.501 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T08:57:44.508 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cmp /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_27.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_27.export 2024-09-14T08:57:44.562 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_27.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_27.export 2024-09-14T08:57:44.594 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:57:44.594 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T08:57:44.594 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_28 2024-09-14T08:57:44.594 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_snap cluster2 mirror image_28 snap 2024-09-14T08:57:44.595 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:57:44.595 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:44.595 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_28 2024-09-14T08:57:44.595 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:57:44.595 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror snap create image_28@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T08:57:45.361 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_started cluster1 mirror image_28 2024-09-14T08:57:45.361 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:57:45.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:45.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_28 2024-09-14T08:57:45.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_state cluster1 mirror image_28 started 2024-09-14T08:57:45.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:57:45.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:45.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_28 2024-09-14T08:57:45.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=started 2024-09-14T08:57:45.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:57:45.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:57:45.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:57:46.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_replay_state cluster1 mirror image_28 started 2024-09-14T08:57:46.363 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:57:46.363 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:46.363 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_28 2024-09-14T08:57:46.363 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=started 2024-09-14T08:57:46.363 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_result 2024-09-14T08:57:46.363 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=stopped 2024-09-14T08:57:46.364 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_28 2024-09-14T08:57:46.364 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_instance=cluster1 2024-09-14T08:57:46.364 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:57:46.364 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ grep -i state 2024-09-14T08:57:46.364 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T08:57:46.364 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance=cluster1 2024-09-14T08:57:46.364 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local loop_instance 2024-09-14T08:57:46.364 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:57:46.364 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sleep 0 2024-09-14T08:57:46.365 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T08:57:46.365 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ seq 0 3 2024-09-14T08:57:46.368 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:57:46.368 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_28 2024-09-14T08:57:46.368 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:0 2024-09-14T08:57:46.368 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:57:46.368 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:57:46.368 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T08:57:46.368 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:0 2024-09-14T08:57:46.368 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:57:46.368 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:57:46.368 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:57:46.369 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=0 2024-09-14T08:57:46.369 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T08:57:46.369 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:57:46.369 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:57:46.369 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=0 2024-09-14T08:57:46.369 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:57:46.369 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T08:57:46.369 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:0 2024-09-14T08:57:46.369 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:57:46.369 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:57:46.369 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T08:57:46.369 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:0 2024-09-14T08:57:46.369 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:57:46.369 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:57:46.370 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:57:46.370 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:57:46.370 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T08:57:46.370 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:57:46.370 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:57:46.370 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=0 2024-09-14T08:57:46.370 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=0 2024-09-14T08:57:46.370 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T08:57:46.388 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:57:46.388 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:57:46.388 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:57:46.388 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_28 2024-09-14T08:57:46.473 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_28'] 2024-09-14T08:57:46.474 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:57:46.474 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:57:46.477 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:57:46.477 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_28 2024-09-14T08:57:46.477 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:1 2024-09-14T08:57:46.477 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:57:46.477 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:57:46.477 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T08:57:46.477 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:1 2024-09-14T08:57:46.478 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:57:46.478 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:57:46.478 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:57:46.478 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=1 2024-09-14T08:57:46.478 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T08:57:46.478 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:57:46.478 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:57:46.478 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=1 2024-09-14T08:57:46.478 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:57:46.478 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T08:57:46.478 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:1 2024-09-14T08:57:46.478 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:57:46.478 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:57:46.478 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T08:57:46.479 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:1 2024-09-14T08:57:46.479 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:57:46.479 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:57:46.479 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:57:46.479 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:57:46.479 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T08:57:46.479 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:57:46.479 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:57:46.479 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=1 2024-09-14T08:57:46.479 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=1 2024-09-14T08:57:46.479 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T08:57:46.497 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:57:46.497 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:57:46.497 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:57:46.497 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_28 2024-09-14T08:57:46.586 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ return 0 2024-09-14T08:57:46.586 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T08:57:46.587 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ echo ' "state": "Replaying"' 2024-09-14T08:57:46.587 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep -i Replaying 2024-09-14T08:57:46.587 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: "state": "Replaying" 2024-09-14T08:57:46.589 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_state=started 2024-09-14T08:57:46.589 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test started = started 2024-09-14T08:57:46.589 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:57:46.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_28 2024-09-14T08:57:46.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:57:46.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:57:46.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:46.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_28 2024-09-14T08:57:46.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = journal ']' 2024-09-14T08:57:46.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = snapshot ']' 2024-09-14T08:57:46.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_28 2024-09-14T08:57:46.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:57:46.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:57:46.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:46.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_28 2024-09-14T08:57:46.591 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster2-mirror-image_28.status 2024-09-14T08:57:46.591 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster2-mirror-image_28.status 2024-09-14T08:57:46.591 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:57:46.591 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_28.status 2024-09-14T08:57:46.591 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster1-mirror-image_28.status 2024-09-14T08:57:46.591 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster1-mirror-image_28.status 2024-09-14T08:57:46.592 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:57:46.592 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_status_log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_28.status 2024-09-14T08:57:46.592 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ mirror_image_snapshot cluster2 mirror image_28 2024-09-14T08:57:46.592 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:57:46.592 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:46.592 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_28 2024-09-14T08:57:46.592 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_28 2024-09-14T08:57:47.366 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Snapshot ID: 140 2024-09-14T08:57:47.372 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_28 /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_28.status 2024-09-14T08:57:47.372 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:57:47.372 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:47.372 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_28 2024-09-14T08:57:47.372 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_28.status 2024-09-14T08:57:47.372 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_28 --format xml 2024-09-14T08:57:47.372 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:57:47.418 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-14T08:57:47.420 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snapshot_id=140 2024-09-14T08:57:47.420 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:57:47.420 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:57:47.420 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.2 2024-09-14T08:57:47.621 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_28 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_28.status 2024-09-14T08:57:47.621 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:57:47.622 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:47.622 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_28 2024-09-14T08:57:47.622 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_28.status 2024-09-14T08:57:47.622 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_28 --format xml 2024-09-14T08:57:47.622 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:57:47.670 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:57:47.670 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:57:47.672 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:57:47.672 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:57:47.672 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:57:47.672 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:57:47.672 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 140 = '' 2024-09-14T08:57:47.672 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:57:47.672 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.4 2024-09-14T08:57:48.073 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_28 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_28.status 2024-09-14T08:57:48.074 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:57:48.074 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:48.074 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_28 2024-09-14T08:57:48.074 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_28.status 2024-09-14T08:57:48.074 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_28 --format xml 2024-09-14T08:57:48.074 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:57:48.121 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:57:48.121 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:57:48.122 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:57:48.123 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:57:48.123 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:57:48.123 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:57:48.123 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 140 = '' 2024-09-14T08:57:48.123 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:57:48.123 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.8 2024-09-14T08:57:48.925 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_28 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_28.status 2024-09-14T08:57:48.925 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:57:48.925 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:48.925 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_28 2024-09-14T08:57:48.925 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_28.status 2024-09-14T08:57:48.925 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_28 --format xml 2024-09-14T08:57:48.925 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:57:48.973 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:57:48.975 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=140 2024-09-14T08:57:48.975 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 140 = 140 2024-09-14T08:57:48.975 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:57:48.975 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snap_present cluster1 mirror image_28 snap 2024-09-14T08:57:48.975 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:57:48.975 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:48.975 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_28 2024-09-14T08:57:48.975 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:57:48.975 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:57:48.975 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T08:57:48.975 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:57:49.976 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 info mirror/image_28@snap 2024-09-14T08:57:50.020 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_28': 2024-09-14T08:57:50.020 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T08:57:50.020 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:57:50.021 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 2 2024-09-14T08:57:50.021 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 12965b63357 2024-09-14T08:57:50.021 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.12965b63357 2024-09-14T08:57:50.021 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:57:50.021 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering, non-primary 2024-09-14T08:57:50.021 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:57:50.021 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:57:50.021 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:49:29 2024 2024-09-14T08:57:50.021 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:49:29 2024 2024-09-14T08:57:50.021 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:49:29 2024 2024-09-14T08:57:50.021 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: protected: False 2024-09-14T08:57:50.021 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:57:50.021 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:57:50.022 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: e5b5824c-8477-465c-bd16-4b286b9a4cc9 2024-09-14T08:57:50.022 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: false 2024-09-14T08:57:50.024 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:57:50.024 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ compare_image_snaps mirror image_28 snap 2024-09-14T08:57:50.024 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:50.024 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_28 2024-09-14T08:57:50.024 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:57:50.024 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local ret=0 2024-09-14T08:57:50.024 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local rmt_export=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_28.export 2024-09-14T08:57:50.024 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local loc_export=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_28.export 2024-09-14T08:57:50.025 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_28.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_28.export 2024-09-14T08:57:50.025 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror export image_28@snap /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_28.export 2024-09-14T08:57:50.210 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T08:57:50.218 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 -p mirror export image_28@snap /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_28.export 2024-09-14T08:57:50.469 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T08:57:50.476 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cmp /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_28.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_28.export 2024-09-14T08:57:50.531 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_28.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_28.export 2024-09-14T08:57:50.563 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:57:50.563 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T08:57:50.563 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_29 2024-09-14T08:57:50.563 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_snap cluster2 mirror image_29 snap 2024-09-14T08:57:50.563 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:57:50.563 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:50.563 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_29 2024-09-14T08:57:50.563 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:57:50.563 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror snap create image_29@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T08:57:51.367 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_started cluster1 mirror image_29 2024-09-14T08:57:51.367 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:57:51.367 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:51.367 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_29 2024-09-14T08:57:51.367 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_state cluster1 mirror image_29 started 2024-09-14T08:57:51.367 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:57:51.368 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:51.368 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_29 2024-09-14T08:57:51.368 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=started 2024-09-14T08:57:51.368 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:57:51.368 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:57:51.368 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:57:52.368 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_replay_state cluster1 mirror image_29 started 2024-09-14T08:57:52.368 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:57:52.369 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:52.369 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_29 2024-09-14T08:57:52.369 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=started 2024-09-14T08:57:52.369 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_result 2024-09-14T08:57:52.369 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=stopped 2024-09-14T08:57:52.370 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_29 2024-09-14T08:57:52.370 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ grep -i state 2024-09-14T08:57:52.370 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_instance=cluster1 2024-09-14T08:57:52.370 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:57:52.370 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T08:57:52.370 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance=cluster1 2024-09-14T08:57:52.370 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local loop_instance 2024-09-14T08:57:52.371 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:57:52.371 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sleep 0 2024-09-14T08:57:52.371 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T08:57:52.371 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ seq 0 3 2024-09-14T08:57:52.372 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:57:52.373 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_29 2024-09-14T08:57:52.373 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:0 2024-09-14T08:57:52.373 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:57:52.373 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:57:52.373 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T08:57:52.373 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:0 2024-09-14T08:57:52.373 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:57:52.373 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:57:52.373 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:57:52.373 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=0 2024-09-14T08:57:52.373 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T08:57:52.373 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:57:52.373 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:57:52.373 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=0 2024-09-14T08:57:52.374 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:57:52.375 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T08:57:52.375 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:0 2024-09-14T08:57:52.375 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:57:52.375 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:57:52.375 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T08:57:52.375 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:0 2024-09-14T08:57:52.375 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:57:52.376 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:57:52.376 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:57:52.376 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:57:52.376 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T08:57:52.376 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:57:52.376 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:57:52.376 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=0 2024-09-14T08:57:52.376 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=0 2024-09-14T08:57:52.376 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T08:57:52.395 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:57:52.395 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:57:52.396 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:57:52.396 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_29 2024-09-14T08:57:52.491 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ return 0 2024-09-14T08:57:52.491 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T08:57:52.491 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ echo ' "state": "Replaying"' 2024-09-14T08:57:52.491 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep -i Replaying 2024-09-14T08:57:52.492 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: "state": "Replaying" 2024-09-14T08:57:52.494 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_state=started 2024-09-14T08:57:52.494 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test started = started 2024-09-14T08:57:52.494 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:57:52.494 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_29 2024-09-14T08:57:52.494 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:57:52.494 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:57:52.494 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:52.494 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_29 2024-09-14T08:57:52.494 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = journal ']' 2024-09-14T08:57:52.494 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = snapshot ']' 2024-09-14T08:57:52.494 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_29 2024-09-14T08:57:52.494 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:57:52.494 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:57:52.495 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:52.495 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_29 2024-09-14T08:57:52.495 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster2-mirror-image_29.status 2024-09-14T08:57:52.495 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster2-mirror-image_29.status 2024-09-14T08:57:52.495 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:57:52.495 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_29.status 2024-09-14T08:57:52.495 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster1-mirror-image_29.status 2024-09-14T08:57:52.496 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster1-mirror-image_29.status 2024-09-14T08:57:52.496 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:57:52.497 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_status_log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_29.status 2024-09-14T08:57:52.497 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ mirror_image_snapshot cluster2 mirror image_29 2024-09-14T08:57:52.497 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:57:52.497 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:52.497 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_29 2024-09-14T08:57:52.497 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_29 2024-09-14T08:57:53.369 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Snapshot ID: 142 2024-09-14T08:57:53.376 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_29 /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_29.status 2024-09-14T08:57:53.376 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:57:53.376 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:53.376 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_29 2024-09-14T08:57:53.376 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_29.status 2024-09-14T08:57:53.377 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_29 --format xml 2024-09-14T08:57:53.377 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:57:53.423 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-14T08:57:53.425 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snapshot_id=142 2024-09-14T08:57:53.425 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:57:53.425 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:57:53.425 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.2 2024-09-14T08:57:53.626 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_29 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_29.status 2024-09-14T08:57:53.627 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:57:53.627 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:53.627 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_29 2024-09-14T08:57:53.627 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_29.status 2024-09-14T08:57:53.627 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_29 --format xml 2024-09-14T08:57:53.627 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:57:53.675 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:57:53.676 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:57:53.677 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:57:53.677 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:57:53.677 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:57:53.678 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:57:53.678 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 142 = '' 2024-09-14T08:57:53.678 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:57:53.678 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.4 2024-09-14T08:57:54.079 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_29 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_29.status 2024-09-14T08:57:54.079 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:57:54.079 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:54.079 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_29 2024-09-14T08:57:54.079 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_29.status 2024-09-14T08:57:54.079 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_29 --format xml 2024-09-14T08:57:54.079 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:57:54.128 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:57:54.128 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:57:54.129 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:57:54.129 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:57:54.129 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:57:54.130 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:57:54.130 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 142 = '' 2024-09-14T08:57:54.130 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:57:54.130 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.8 2024-09-14T08:57:54.931 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_29 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_29.status 2024-09-14T08:57:54.931 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:57:54.931 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:54.932 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_29 2024-09-14T08:57:54.932 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_29.status 2024-09-14T08:57:54.932 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_29 --format xml 2024-09-14T08:57:54.932 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:57:54.980 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:57:54.982 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=142 2024-09-14T08:57:54.982 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 142 = 142 2024-09-14T08:57:54.982 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:57:54.982 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snap_present cluster1 mirror image_29 snap 2024-09-14T08:57:54.982 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:57:54.982 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:54.982 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_29 2024-09-14T08:57:54.982 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:57:54.982 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:57:54.983 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T08:57:54.983 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:57:55.983 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 info mirror/image_29@snap 2024-09-14T08:57:56.030 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_29': 2024-09-14T08:57:56.030 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T08:57:56.030 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:57:56.030 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 2 2024-09-14T08:57:56.030 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 10bd8d8a2ee1 2024-09-14T08:57:56.030 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.10bd8d8a2ee1 2024-09-14T08:57:56.031 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:57:56.031 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering, non-primary 2024-09-14T08:57:56.031 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:57:56.031 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:57:56.031 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:49:42 2024 2024-09-14T08:57:56.031 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:49:42 2024 2024-09-14T08:57:56.031 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:49:42 2024 2024-09-14T08:57:56.031 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: protected: False 2024-09-14T08:57:56.031 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:57:56.031 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:57:56.031 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: 50c4f65a-9027-4112-bc18-18bc2d737492 2024-09-14T08:57:56.031 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: false 2024-09-14T08:57:56.034 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:57:56.034 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ compare_image_snaps mirror image_29 snap 2024-09-14T08:57:56.034 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:56.034 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_29 2024-09-14T08:57:56.034 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:57:56.034 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local ret=0 2024-09-14T08:57:56.034 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local rmt_export=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_29.export 2024-09-14T08:57:56.035 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local loc_export=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_29.export 2024-09-14T08:57:56.035 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_29.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_29.export 2024-09-14T08:57:56.035 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror export image_29@snap /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_29.export 2024-09-14T08:57:56.222 INFO:tasks.ceph.cluster2.osd.2.smithi164.stderr:2024-09-14T08:57:56.222+0000 7fe32301f640 -1 reset not still connected to 0x55e770c148f0 2024-09-14T08:57:56.235 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T08:57:56.242 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 -p mirror export image_29@snap /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_29.export 2024-09-14T08:57:56.497 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T08:57:56.504 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cmp /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_29.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_29.export 2024-09-14T08:57:56.559 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_29.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_29.export 2024-09-14T08:57:56.591 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:57:56.591 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T08:57:56.591 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_30 2024-09-14T08:57:56.591 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_snap cluster2 mirror image_30 snap 2024-09-14T08:57:56.591 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:57:56.591 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:56.591 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_30 2024-09-14T08:57:56.591 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:57:56.591 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror snap create image_30@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T08:57:57.375 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_started cluster1 mirror image_30 2024-09-14T08:57:57.375 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:57:57.375 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:57.375 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_30 2024-09-14T08:57:57.375 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_state cluster1 mirror image_30 started 2024-09-14T08:57:57.375 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:57:57.375 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:57.375 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_30 2024-09-14T08:57:57.375 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=started 2024-09-14T08:57:57.376 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:57:57.376 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:57:57.376 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:57:58.376 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_replay_state cluster1 mirror image_30 started 2024-09-14T08:57:58.376 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:57:58.376 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:58.376 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_30 2024-09-14T08:57:58.376 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=started 2024-09-14T08:57:58.376 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_result 2024-09-14T08:57:58.377 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=stopped 2024-09-14T08:57:58.378 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_30 2024-09-14T08:57:58.378 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_instance=cluster1 2024-09-14T08:57:58.378 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ grep -i state 2024-09-14T08:57:58.378 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:57:58.378 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T08:57:58.378 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance=cluster1 2024-09-14T08:57:58.378 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local loop_instance 2024-09-14T08:57:58.378 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:57:58.378 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sleep 0 2024-09-14T08:57:58.378 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T08:57:58.381 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ seq 0 3 2024-09-14T08:57:58.381 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:57:58.381 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_30 2024-09-14T08:57:58.381 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:0 2024-09-14T08:57:58.381 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:57:58.381 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:57:58.381 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T08:57:58.381 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:0 2024-09-14T08:57:58.381 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:57:58.382 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:57:58.382 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:57:58.382 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=0 2024-09-14T08:57:58.382 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T08:57:58.382 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:57:58.382 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:57:58.382 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=0 2024-09-14T08:57:58.382 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:57:58.382 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T08:57:58.382 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:0 2024-09-14T08:57:58.382 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:57:58.383 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:57:58.383 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T08:57:58.383 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:0 2024-09-14T08:57:58.383 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:57:58.383 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:57:58.383 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:57:58.383 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:57:58.383 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T08:57:58.383 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:57:58.383 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:57:58.383 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=0 2024-09-14T08:57:58.383 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=0 2024-09-14T08:57:58.383 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T08:57:58.404 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:57:58.404 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:57:58.404 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:57:58.404 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_30 2024-09-14T08:57:58.499 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ return 0 2024-09-14T08:57:58.500 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T08:57:58.500 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ echo ' "state": "Replaying"' 2024-09-14T08:57:58.500 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep -i Replaying 2024-09-14T08:57:58.501 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: "state": "Replaying" 2024-09-14T08:57:58.502 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_state=started 2024-09-14T08:57:58.502 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test started = started 2024-09-14T08:57:58.503 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:57:58.503 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_30 2024-09-14T08:57:58.503 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:57:58.503 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:57:58.503 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:58.503 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_30 2024-09-14T08:57:58.503 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = journal ']' 2024-09-14T08:57:58.503 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = snapshot ']' 2024-09-14T08:57:58.503 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_30 2024-09-14T08:57:58.503 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:57:58.503 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:57:58.503 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:58.503 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_30 2024-09-14T08:57:58.504 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster2-mirror-image_30.status 2024-09-14T08:57:58.504 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster2-mirror-image_30.status 2024-09-14T08:57:58.504 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:57:58.505 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_30.status 2024-09-14T08:57:58.505 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster1-mirror-image_30.status 2024-09-14T08:57:58.505 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster1-mirror-image_30.status 2024-09-14T08:57:58.505 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:57:58.505 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_status_log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_30.status 2024-09-14T08:57:58.505 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ mirror_image_snapshot cluster2 mirror image_30 2024-09-14T08:57:58.505 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:57:58.506 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:58.506 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_30 2024-09-14T08:57:58.506 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_30 2024-09-14T08:57:59.380 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Snapshot ID: 144 2024-09-14T08:57:59.387 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_30 /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_30.status 2024-09-14T08:57:59.387 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:57:59.387 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:59.387 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_30 2024-09-14T08:57:59.387 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_30.status 2024-09-14T08:57:59.387 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_30 --format xml 2024-09-14T08:57:59.387 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:57:59.433 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-14T08:57:59.435 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snapshot_id=144 2024-09-14T08:57:59.435 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:57:59.435 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:57:59.435 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.2 2024-09-14T08:57:59.636 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_30 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_30.status 2024-09-14T08:57:59.636 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:57:59.636 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:57:59.637 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_30 2024-09-14T08:57:59.637 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_30.status 2024-09-14T08:57:59.637 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_30 --format xml 2024-09-14T08:57:59.637 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:57:59.688 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:57:59.688 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:57:59.690 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:57:59.690 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:57:59.690 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:57:59.690 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:57:59.690 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 144 = '' 2024-09-14T08:57:59.691 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:57:59.691 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.4 2024-09-14T08:58:00.092 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_30 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_30.status 2024-09-14T08:58:00.092 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:58:00.092 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:00.092 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_30 2024-09-14T08:58:00.092 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_30.status 2024-09-14T08:58:00.092 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_30 --format xml 2024-09-14T08:58:00.092 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:58:00.141 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:58:00.141 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:58:00.143 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:58:00.143 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:58:00.143 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:58:00.143 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:58:00.143 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 144 = '' 2024-09-14T08:58:00.143 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:58:00.143 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.8 2024-09-14T08:58:00.944 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.7408371396570209) > weight (0.25) 2024-09-14T08:58:00.945 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.9827686185899677) > weight (0.25) 2024-09-14T08:58:00.945 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.9447173814376943) > weight (0.25) 2024-09-14T08:58:00.945 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.3 with skip (0.9683733121586059) > weight (0.25) 2024-09-14T08:58:00.945 INFO:tasks.rbd_mirror_thrash:waiting for 119 secs before thrashing 2024-09-14T08:58:00.946 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_30 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_30.status 2024-09-14T08:58:00.946 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:58:00.946 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:00.946 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_30 2024-09-14T08:58:00.946 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_30.status 2024-09-14T08:58:00.946 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_30 --format xml 2024-09-14T08:58:00.946 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:58:00.995 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:58:00.997 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=144 2024-09-14T08:58:00.998 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 144 = 144 2024-09-14T08:58:00.998 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:58:00.998 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snap_present cluster1 mirror image_30 snap 2024-09-14T08:58:00.998 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:58:00.998 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:00.998 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_30 2024-09-14T08:58:00.998 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:58:00.998 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:58:00.998 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T08:58:00.998 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:58:01.998 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 info mirror/image_30@snap 2024-09-14T08:58:02.044 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_30': 2024-09-14T08:58:02.044 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T08:58:02.044 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:58:02.044 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 2 2024-09-14T08:58:02.044 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 1189c563b189 2024-09-14T08:58:02.044 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.1189c563b189 2024-09-14T08:58:02.044 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:58:02.044 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering, non-primary 2024-09-14T08:58:02.044 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:58:02.044 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:58:02.045 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:49:54 2024 2024-09-14T08:58:02.045 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:49:54 2024 2024-09-14T08:58:02.045 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:49:54 2024 2024-09-14T08:58:02.045 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: protected: False 2024-09-14T08:58:02.045 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:58:02.045 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:58:02.045 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: 4e345c40-e13b-4117-8365-22395b48e4b8 2024-09-14T08:58:02.045 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: false 2024-09-14T08:58:02.048 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:58:02.048 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ compare_image_snaps mirror image_30 snap 2024-09-14T08:58:02.048 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:02.048 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_30 2024-09-14T08:58:02.048 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:58:02.048 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local ret=0 2024-09-14T08:58:02.048 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local rmt_export=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_30.export 2024-09-14T08:58:02.048 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local loc_export=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_30.export 2024-09-14T08:58:02.048 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_30.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_30.export 2024-09-14T08:58:02.049 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror export image_30@snap /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_30.export 2024-09-14T08:58:02.252 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T08:58:02.259 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 -p mirror export image_30@snap /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_30.export 2024-09-14T08:58:02.515 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T08:58:02.522 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cmp /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_30.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_30.export 2024-09-14T08:58:02.576 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_30.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_30.export 2024-09-14T08:58:02.608 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:58:02.608 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T08:58:02.608 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_31 2024-09-14T08:58:02.608 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_snap cluster2 mirror image_31 snap 2024-09-14T08:58:02.608 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:58:02.608 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:02.608 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_31 2024-09-14T08:58:02.609 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:58:02.609 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror snap create image_31@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T08:58:03.381 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_started cluster1 mirror image_31 2024-09-14T08:58:03.381 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:58:03.382 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:03.382 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_31 2024-09-14T08:58:03.382 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_state cluster1 mirror image_31 started 2024-09-14T08:58:03.382 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:58:03.382 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:03.382 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_31 2024-09-14T08:58:03.382 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=started 2024-09-14T08:58:03.382 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:58:03.382 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:58:03.382 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:58:04.382 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_replay_state cluster1 mirror image_31 started 2024-09-14T08:58:04.383 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:58:04.383 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:04.383 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_31 2024-09-14T08:58:04.383 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=started 2024-09-14T08:58:04.383 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_result 2024-09-14T08:58:04.383 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=stopped 2024-09-14T08:58:04.384 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_31 2024-09-14T08:58:04.384 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_instance=cluster1 2024-09-14T08:58:04.384 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:58:04.384 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ grep -i state 2024-09-14T08:58:04.384 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T08:58:04.384 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance=cluster1 2024-09-14T08:58:04.385 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local loop_instance 2024-09-14T08:58:04.385 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:58:04.385 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sleep 0 2024-09-14T08:58:04.385 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T08:58:04.388 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ seq 0 3 2024-09-14T08:58:04.388 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:58:04.388 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_31 2024-09-14T08:58:04.388 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:0 2024-09-14T08:58:04.388 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:58:04.388 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:58:04.388 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T08:58:04.388 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:0 2024-09-14T08:58:04.388 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:58:04.389 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:58:04.389 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:58:04.389 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=0 2024-09-14T08:58:04.389 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T08:58:04.389 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:58:04.389 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:58:04.389 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=0 2024-09-14T08:58:04.389 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:58:04.389 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T08:58:04.389 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:0 2024-09-14T08:58:04.389 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:58:04.389 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:58:04.389 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T08:58:04.389 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:0 2024-09-14T08:58:04.390 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:58:04.390 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:58:04.390 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:58:04.390 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:58:04.390 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T08:58:04.390 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:58:04.390 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:58:04.390 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=0 2024-09-14T08:58:04.390 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=0 2024-09-14T08:58:04.390 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T08:58:04.409 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:58:04.409 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:58:04.409 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:58:04.409 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_31 2024-09-14T08:58:04.497 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_31'] 2024-09-14T08:58:04.497 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:58:04.497 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:58:04.501 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:58:04.501 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_31 2024-09-14T08:58:04.501 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:1 2024-09-14T08:58:04.501 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:58:04.501 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:58:04.501 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T08:58:04.501 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:1 2024-09-14T08:58:04.501 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:58:04.501 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:58:04.501 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:58:04.501 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=1 2024-09-14T08:58:04.501 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T08:58:04.502 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:58:04.502 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:58:04.502 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=1 2024-09-14T08:58:04.502 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:58:04.502 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T08:58:04.502 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:1 2024-09-14T08:58:04.502 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:58:04.502 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:58:04.502 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T08:58:04.502 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:1 2024-09-14T08:58:04.502 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:58:04.502 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:58:04.502 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:58:04.502 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:58:04.502 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T08:58:04.503 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:58:04.503 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:58:04.503 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=1 2024-09-14T08:58:04.503 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=1 2024-09-14T08:58:04.503 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T08:58:04.519 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:58:04.519 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:58:04.519 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:58:04.519 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_31 2024-09-14T08:58:04.601 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_31'] 2024-09-14T08:58:04.601 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:58:04.601 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:58:04.604 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:58:04.604 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_31 2024-09-14T08:58:04.604 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:2 2024-09-14T08:58:04.604 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:58:04.604 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:58:04.604 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-14T08:58:04.605 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:2 2024-09-14T08:58:04.605 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:58:04.605 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:58:04.605 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:58:04.605 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=2 2024-09-14T08:58:04.605 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T08:58:04.605 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:58:04.605 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:58:04.605 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=2 2024-09-14T08:58:04.605 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:58:04.605 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-14T08:58:04.605 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:2 2024-09-14T08:58:04.605 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:58:04.605 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:58:04.606 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T08:58:04.606 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:2 2024-09-14T08:58:04.606 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:58:04.606 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:58:04.606 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:58:04.606 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:58:04.606 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-14T08:58:04.606 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:58:04.606 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:58:04.606 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=2 2024-09-14T08:58:04.606 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=2 2024-09-14T08:58:04.606 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T08:58:04.624 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:58:04.624 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:58:04.624 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:58:04.624 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_31 2024-09-14T08:58:04.707 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_31'] 2024-09-14T08:58:04.707 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:58:04.707 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:58:04.710 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:58:04.710 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_31 2024-09-14T08:58:04.710 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:3 2024-09-14T08:58:04.710 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:58:04.710 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:58:04.710 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:3 cluster instance 2024-09-14T08:58:04.710 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:3 2024-09-14T08:58:04.710 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:58:04.710 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:58:04.711 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:58:04.711 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=3 2024-09-14T08:58:04.711 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-14T08:58:04.711 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:58:04.711 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:58:04.711 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=3 2024-09-14T08:58:04.711 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=3 2024-09-14T08:58:04.711 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:3 cluster1 2024-09-14T08:58:04.711 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:3 2024-09-14T08:58:04.711 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:58:04.711 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:58:04.711 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-14T08:58:04.711 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:3 2024-09-14T08:58:04.711 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:58:04.712 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:58:04.712 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:58:04.712 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=3 2024-09-14T08:58:04.712 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 3 = cluster1:3 ']' 2024-09-14T08:58:04.712 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:58:04.712 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:58:04.712 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=3 2024-09-14T08:58:04.712 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=3 2024-09-14T08:58:04.712 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-14T08:58:04.730 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T08:58:04.730 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T08:58:04.730 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T08:58:04.730 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_31 2024-09-14T08:58:04.819 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ return 0 2024-09-14T08:58:04.820 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T08:58:04.820 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ echo ' "state": "Replaying"' 2024-09-14T08:58:04.820 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep -i Replaying 2024-09-14T08:58:04.821 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: "state": "Replaying" 2024-09-14T08:58:04.822 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_state=started 2024-09-14T08:58:04.822 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test started = started 2024-09-14T08:58:04.823 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:58:04.823 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_31 2024-09-14T08:58:04.823 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:58:04.823 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:58:04.823 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:04.823 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_31 2024-09-14T08:58:04.823 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = journal ']' 2024-09-14T08:58:04.823 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = snapshot ']' 2024-09-14T08:58:04.823 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_31 2024-09-14T08:58:04.823 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:58:04.823 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:58:04.823 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:04.823 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_31 2024-09-14T08:58:04.823 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster2-mirror-image_31.status 2024-09-14T08:58:04.824 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster2-mirror-image_31.status 2024-09-14T08:58:04.824 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:58:04.824 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_31.status 2024-09-14T08:58:04.825 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster1-mirror-image_31.status 2024-09-14T08:58:04.825 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster1-mirror-image_31.status 2024-09-14T08:58:04.825 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:58:04.825 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_status_log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_31.status 2024-09-14T08:58:04.825 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ mirror_image_snapshot cluster2 mirror image_31 2024-09-14T08:58:04.825 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:58:04.825 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:04.825 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_31 2024-09-14T08:58:04.825 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_31 2024-09-14T08:58:05.386 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Snapshot ID: 146 2024-09-14T08:58:05.392 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_31 /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_31.status 2024-09-14T08:58:05.392 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:58:05.392 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:05.392 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_31 2024-09-14T08:58:05.392 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_31.status 2024-09-14T08:58:05.392 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_31 --format xml 2024-09-14T08:58:05.392 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:58:05.438 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-14T08:58:05.440 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snapshot_id=146 2024-09-14T08:58:05.440 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:58:05.440 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:58:05.440 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.2 2024-09-14T08:58:05.641 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_31 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_31.status 2024-09-14T08:58:05.642 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:58:05.642 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:05.642 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_31 2024-09-14T08:58:05.642 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_31.status 2024-09-14T08:58:05.642 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_31 --format xml 2024-09-14T08:58:05.642 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:58:05.689 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:58:05.689 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:58:05.690 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:58:05.690 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:58:05.691 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:58:05.691 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:58:05.691 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 146 = '' 2024-09-14T08:58:05.691 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:58:05.691 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.4 2024-09-14T08:58:06.092 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_31 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_31.status 2024-09-14T08:58:06.093 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:58:06.093 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:06.093 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_31 2024-09-14T08:58:06.093 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_31.status 2024-09-14T08:58:06.093 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_31 --format xml 2024-09-14T08:58:06.093 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:58:06.143 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:58:06.144 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:58:06.145 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:58:06.145 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:58:06.145 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:58:06.146 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:58:06.146 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 146 = '' 2024-09-14T08:58:06.146 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:58:06.146 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.8 2024-09-14T08:58:06.947 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_31 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_31.status 2024-09-14T08:58:06.947 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:58:06.947 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:06.947 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_31 2024-09-14T08:58:06.947 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_31.status 2024-09-14T08:58:06.947 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_31 --format xml 2024-09-14T08:58:06.947 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:58:06.995 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:58:06.997 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=146 2024-09-14T08:58:06.997 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 146 = 146 2024-09-14T08:58:06.998 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:58:06.998 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snap_present cluster1 mirror image_31 snap 2024-09-14T08:58:06.998 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:58:06.998 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:06.998 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_31 2024-09-14T08:58:06.998 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:58:06.998 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:58:06.999 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T08:58:06.999 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:58:07.998 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 info mirror/image_31@snap 2024-09-14T08:58:08.045 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_31': 2024-09-14T08:58:08.045 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T08:58:08.045 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:58:08.045 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 2 2024-09-14T08:58:08.045 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 12a0e3cf6602 2024-09-14T08:58:08.045 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.12a0e3cf6602 2024-09-14T08:58:08.045 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:58:08.045 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering, non-primary 2024-09-14T08:58:08.045 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:58:08.046 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:58:08.046 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:50:28 2024 2024-09-14T08:58:08.046 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:50:28 2024 2024-09-14T08:58:08.046 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:50:28 2024 2024-09-14T08:58:08.046 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: protected: False 2024-09-14T08:58:08.046 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:58:08.046 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:58:08.046 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: ca4b7bbf-4e51-4087-ac80-e20897556ab1 2024-09-14T08:58:08.046 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: false 2024-09-14T08:58:08.049 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:58:08.049 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ compare_image_snaps mirror image_31 snap 2024-09-14T08:58:08.049 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:08.049 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_31 2024-09-14T08:58:08.049 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:58:08.049 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local ret=0 2024-09-14T08:58:08.050 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local rmt_export=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_31.export 2024-09-14T08:58:08.050 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local loc_export=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_31.export 2024-09-14T08:58:08.050 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_31.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_31.export 2024-09-14T08:58:08.050 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror export image_31@snap /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_31.export 2024-09-14T08:58:08.239 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T08:58:08.247 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 -p mirror export image_31@snap /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_31.export 2024-09-14T08:58:08.546 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T08:58:08.553 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cmp /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_31.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_31.export 2024-09-14T08:58:08.608 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_31.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_31.export 2024-09-14T08:58:08.640 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:58:08.640 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T08:58:08.640 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_32 2024-09-14T08:58:08.640 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_snap cluster2 mirror image_32 snap 2024-09-14T08:58:08.640 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:58:08.640 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:08.640 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_32 2024-09-14T08:58:08.640 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:58:08.640 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror snap create image_32@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T08:58:09.400 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_started cluster1 mirror image_32 2024-09-14T08:58:09.400 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:58:09.400 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:09.400 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_32 2024-09-14T08:58:09.400 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_state cluster1 mirror image_32 started 2024-09-14T08:58:09.400 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:58:09.400 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:09.401 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_32 2024-09-14T08:58:09.401 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=started 2024-09-14T08:58:09.401 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:58:09.401 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:58:09.401 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:58:10.402 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_replay_state cluster1 mirror image_32 started 2024-09-14T08:58:10.402 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:58:10.402 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:10.402 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_32 2024-09-14T08:58:10.402 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=started 2024-09-14T08:58:10.402 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_result 2024-09-14T08:58:10.403 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=stopped 2024-09-14T08:58:10.403 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_32 2024-09-14T08:58:10.403 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_instance=cluster1 2024-09-14T08:58:10.403 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:58:10.403 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T08:58:10.403 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance=cluster1 2024-09-14T08:58:10.403 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local loop_instance 2024-09-14T08:58:10.403 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ grep -i state 2024-09-14T08:58:10.403 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:58:10.403 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sleep 0 2024-09-14T08:58:10.406 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T08:58:10.406 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ seq 0 3 2024-09-14T08:58:10.406 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:58:10.406 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_32 2024-09-14T08:58:10.406 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:0 2024-09-14T08:58:10.406 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:58:10.406 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:58:10.407 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T08:58:10.407 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:0 2024-09-14T08:58:10.407 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:58:10.407 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:58:10.407 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:58:10.407 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=0 2024-09-14T08:58:10.407 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T08:58:10.407 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:58:10.407 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:58:10.407 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=0 2024-09-14T08:58:10.407 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:58:10.407 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T08:58:10.407 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:0 2024-09-14T08:58:10.407 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:58:10.407 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:58:10.408 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T08:58:10.408 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:0 2024-09-14T08:58:10.408 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:58:10.408 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:58:10.408 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:58:10.408 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:58:10.408 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T08:58:10.408 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:58:10.408 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:58:10.408 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=0 2024-09-14T08:58:10.408 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=0 2024-09-14T08:58:10.408 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T08:58:10.427 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:58:10.427 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:58:10.427 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:58:10.427 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_32 2024-09-14T08:58:10.515 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_32'] 2024-09-14T08:58:10.515 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:58:10.515 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:58:10.518 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:58:10.518 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_32 2024-09-14T08:58:10.518 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:1 2024-09-14T08:58:10.518 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:58:10.518 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:58:10.518 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T08:58:10.519 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:1 2024-09-14T08:58:10.519 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:58:10.519 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:58:10.519 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:58:10.519 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=1 2024-09-14T08:58:10.519 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T08:58:10.519 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:58:10.519 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:58:10.519 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=1 2024-09-14T08:58:10.519 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:58:10.519 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T08:58:10.519 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:1 2024-09-14T08:58:10.519 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:58:10.519 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:58:10.519 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T08:58:10.519 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:1 2024-09-14T08:58:10.520 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:58:10.520 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:58:10.520 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:58:10.520 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:58:10.520 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T08:58:10.520 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:58:10.520 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:58:10.520 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=1 2024-09-14T08:58:10.520 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=1 2024-09-14T08:58:10.520 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T08:58:10.536 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:58:10.536 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:58:10.536 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:58:10.536 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_32 2024-09-14T08:58:10.626 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ return 0 2024-09-14T08:58:10.626 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T08:58:10.626 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ echo ' "state": "Replaying"' 2024-09-14T08:58:10.626 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep -i Replaying 2024-09-14T08:58:10.627 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: "state": "Replaying" 2024-09-14T08:58:10.628 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_state=started 2024-09-14T08:58:10.628 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test started = started 2024-09-14T08:58:10.628 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:58:10.629 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_32 2024-09-14T08:58:10.629 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:58:10.629 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:58:10.629 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:10.629 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_32 2024-09-14T08:58:10.629 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = journal ']' 2024-09-14T08:58:10.629 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = snapshot ']' 2024-09-14T08:58:10.629 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_32 2024-09-14T08:58:10.629 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:58:10.629 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:58:10.629 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:10.629 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_32 2024-09-14T08:58:10.629 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster2-mirror-image_32.status 2024-09-14T08:58:10.630 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster2-mirror-image_32.status 2024-09-14T08:58:10.630 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:58:10.631 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_32.status 2024-09-14T08:58:10.631 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster1-mirror-image_32.status 2024-09-14T08:58:10.631 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster1-mirror-image_32.status 2024-09-14T08:58:10.631 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:58:10.631 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_status_log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_32.status 2024-09-14T08:58:10.631 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ mirror_image_snapshot cluster2 mirror image_32 2024-09-14T08:58:10.631 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:58:10.631 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:10.631 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_32 2024-09-14T08:58:10.631 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_32 2024-09-14T08:58:11.406 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Snapshot ID: 148 2024-09-14T08:58:11.413 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_32 /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_32.status 2024-09-14T08:58:11.413 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:58:11.413 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:11.413 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_32 2024-09-14T08:58:11.414 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_32.status 2024-09-14T08:58:11.414 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_32 --format xml 2024-09-14T08:58:11.414 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:58:11.459 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-14T08:58:11.461 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snapshot_id=148 2024-09-14T08:58:11.461 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:58:11.461 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:58:11.461 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.2 2024-09-14T08:58:11.663 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_32 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_32.status 2024-09-14T08:58:11.663 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:58:11.664 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:11.664 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_32 2024-09-14T08:58:11.664 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_32.status 2024-09-14T08:58:11.664 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_32 --format xml 2024-09-14T08:58:11.664 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:58:11.711 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:58:11.713 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=66 2024-09-14T08:58:11.713 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 148 = 66 2024-09-14T08:58:11.713 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:58:11.713 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.4 2024-09-14T08:58:12.114 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_32 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_32.status 2024-09-14T08:58:12.114 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:58:12.114 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:12.114 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_32 2024-09-14T08:58:12.114 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_32.status 2024-09-14T08:58:12.114 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_32 --format xml 2024-09-14T08:58:12.114 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:58:12.163 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:58:12.165 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=66 2024-09-14T08:58:12.165 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 148 = 66 2024-09-14T08:58:12.165 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:58:12.165 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.8 2024-09-14T08:58:12.966 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_32 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_32.status 2024-09-14T08:58:12.966 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:58:12.966 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:12.966 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_32 2024-09-14T08:58:12.966 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_32.status 2024-09-14T08:58:12.966 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_32 --format xml 2024-09-14T08:58:12.966 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:58:13.017 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:58:13.017 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:58:13.019 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:58:13.019 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:58:13.019 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:58:13.019 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:58:13.019 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 148 = '' 2024-09-14T08:58:13.019 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:58:13.019 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1.6 2024-09-14T08:58:14.621 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_32 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_32.status 2024-09-14T08:58:14.621 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:58:14.621 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:14.621 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_32 2024-09-14T08:58:14.621 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_32.status 2024-09-14T08:58:14.621 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_32 --format xml 2024-09-14T08:58:14.621 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:58:14.671 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:58:14.673 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=148 2024-09-14T08:58:14.673 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 148 = 148 2024-09-14T08:58:14.673 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:58:14.673 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snap_present cluster1 mirror image_32 snap 2024-09-14T08:58:14.673 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:58:14.673 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:14.673 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_32 2024-09-14T08:58:14.673 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:58:14.673 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:58:14.674 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T08:58:14.674 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:58:15.674 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 info mirror/image_32@snap 2024-09-14T08:58:15.721 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_32': 2024-09-14T08:58:15.721 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T08:58:15.722 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:58:15.722 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 2 2024-09-14T08:58:15.722 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 12a0d9c4cc54 2024-09-14T08:58:15.722 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.12a0d9c4cc54 2024-09-14T08:58:15.722 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:58:15.722 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering, non-primary 2024-09-14T08:58:15.722 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:58:15.722 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:58:15.722 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:50:17 2024 2024-09-14T08:58:15.722 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:50:17 2024 2024-09-14T08:58:15.722 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:50:17 2024 2024-09-14T08:58:15.722 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: protected: False 2024-09-14T08:58:15.722 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:58:15.722 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:58:15.723 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: 7da0399c-f6eb-473e-a82d-0b57a2ea8a70 2024-09-14T08:58:15.723 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: false 2024-09-14T08:58:15.725 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:58:15.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ compare_image_snaps mirror image_32 snap 2024-09-14T08:58:15.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:15.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_32 2024-09-14T08:58:15.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:58:15.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local ret=0 2024-09-14T08:58:15.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local rmt_export=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_32.export 2024-09-14T08:58:15.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local loc_export=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_32.export 2024-09-14T08:58:15.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_32.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_32.export 2024-09-14T08:58:15.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror export image_32@snap /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_32.export 2024-09-14T08:58:15.938 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T08:58:15.944 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 -p mirror export image_32@snap /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_32.export 2024-09-14T08:58:16.146 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T08:58:16.153 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cmp /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_32.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_32.export 2024-09-14T08:58:16.208 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_32.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_32.export 2024-09-14T08:58:16.239 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:58:16.240 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T08:58:16.240 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_33 2024-09-14T08:58:16.240 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_snap cluster2 mirror image_33 snap 2024-09-14T08:58:16.240 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:58:16.240 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:16.240 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_33 2024-09-14T08:58:16.240 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:58:16.240 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror snap create image_33@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T08:58:16.404 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_started cluster1 mirror image_33 2024-09-14T08:58:16.405 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:58:16.405 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:16.405 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_33 2024-09-14T08:58:16.405 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_state cluster1 mirror image_33 started 2024-09-14T08:58:16.405 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:58:16.405 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:16.405 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_33 2024-09-14T08:58:16.405 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=started 2024-09-14T08:58:16.405 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:58:16.405 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:58:16.405 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:58:17.405 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_replay_state cluster1 mirror image_33 started 2024-09-14T08:58:17.406 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:58:17.406 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:17.406 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_33 2024-09-14T08:58:17.406 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=started 2024-09-14T08:58:17.406 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_result 2024-09-14T08:58:17.406 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=stopped 2024-09-14T08:58:17.407 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_33 2024-09-14T08:58:17.407 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_instance=cluster1 2024-09-14T08:58:17.407 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:58:17.407 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T08:58:17.408 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance=cluster1 2024-09-14T08:58:17.408 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local loop_instance 2024-09-14T08:58:17.408 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ grep -i state 2024-09-14T08:58:17.408 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:58:17.408 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sleep 0 2024-09-14T08:58:17.408 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T08:58:17.408 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ seq 0 3 2024-09-14T08:58:17.412 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:58:17.412 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_33 2024-09-14T08:58:17.412 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:0 2024-09-14T08:58:17.412 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:58:17.412 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:58:17.412 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T08:58:17.413 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:0 2024-09-14T08:58:17.413 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:58:17.413 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:58:17.413 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:58:17.413 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=0 2024-09-14T08:58:17.413 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T08:58:17.413 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:58:17.413 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:58:17.413 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=0 2024-09-14T08:58:17.413 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:58:17.413 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T08:58:17.413 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:0 2024-09-14T08:58:17.413 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:58:17.413 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:58:17.414 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T08:58:17.414 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:0 2024-09-14T08:58:17.414 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:58:17.414 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:58:17.414 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:58:17.414 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:58:17.414 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T08:58:17.414 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:58:17.414 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:58:17.414 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=0 2024-09-14T08:58:17.414 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=0 2024-09-14T08:58:17.414 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T08:58:17.431 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:58:17.431 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:58:17.431 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:58:17.431 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_33 2024-09-14T08:58:17.517 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_33'] 2024-09-14T08:58:17.518 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:58:17.518 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:58:17.522 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:58:17.522 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_33 2024-09-14T08:58:17.522 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:1 2024-09-14T08:58:17.522 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:58:17.522 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:58:17.522 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T08:58:17.523 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:1 2024-09-14T08:58:17.523 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:58:17.523 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:58:17.523 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:58:17.523 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=1 2024-09-14T08:58:17.523 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T08:58:17.523 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:58:17.523 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:58:17.524 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=1 2024-09-14T08:58:17.524 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:58:17.524 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T08:58:17.524 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:1 2024-09-14T08:58:17.524 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:58:17.524 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:58:17.524 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T08:58:17.524 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:1 2024-09-14T08:58:17.525 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:58:17.525 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:58:17.525 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:58:17.525 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:58:17.525 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T08:58:17.525 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:58:17.525 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:58:17.525 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=1 2024-09-14T08:58:17.526 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=1 2024-09-14T08:58:17.526 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T08:58:17.539 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:58:17.539 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:58:17.539 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:58:17.539 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_33 2024-09-14T08:58:17.621 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_33'] 2024-09-14T08:58:17.621 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:58:17.621 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:58:17.624 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:58:17.624 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_33 2024-09-14T08:58:17.624 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:2 2024-09-14T08:58:17.624 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:58:17.624 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:58:17.624 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-14T08:58:17.624 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:2 2024-09-14T08:58:17.624 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:58:17.624 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:58:17.624 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:58:17.624 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=2 2024-09-14T08:58:17.625 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T08:58:17.625 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:58:17.625 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:58:17.625 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=2 2024-09-14T08:58:17.625 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:58:17.625 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-14T08:58:17.625 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:2 2024-09-14T08:58:17.625 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:58:17.625 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:58:17.625 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T08:58:17.625 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:2 2024-09-14T08:58:17.625 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:58:17.625 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:58:17.625 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:58:17.626 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:58:17.626 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-14T08:58:17.626 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:58:17.626 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:58:17.626 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=2 2024-09-14T08:58:17.626 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=2 2024-09-14T08:58:17.626 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T08:58:17.642 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:58:17.642 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:58:17.642 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:58:17.642 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_33 2024-09-14T08:58:17.732 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ return 0 2024-09-14T08:58:17.733 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T08:58:17.733 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ echo ' "state": "Replaying"' 2024-09-14T08:58:17.733 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep -i Replaying 2024-09-14T08:58:17.734 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: "state": "Replaying" 2024-09-14T08:58:17.735 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_state=started 2024-09-14T08:58:17.735 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test started = started 2024-09-14T08:58:17.735 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:58:17.735 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_33 2024-09-14T08:58:17.735 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:58:17.736 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:58:17.736 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:17.736 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_33 2024-09-14T08:58:17.736 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = journal ']' 2024-09-14T08:58:17.736 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = snapshot ']' 2024-09-14T08:58:17.736 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_33 2024-09-14T08:58:17.736 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:58:17.736 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:58:17.736 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:17.736 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_33 2024-09-14T08:58:17.736 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster2-mirror-image_33.status 2024-09-14T08:58:17.737 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster2-mirror-image_33.status 2024-09-14T08:58:17.737 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:58:17.738 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_33.status 2024-09-14T08:58:17.738 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster1-mirror-image_33.status 2024-09-14T08:58:17.738 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster1-mirror-image_33.status 2024-09-14T08:58:17.738 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:58:17.738 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_status_log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_33.status 2024-09-14T08:58:17.738 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ mirror_image_snapshot cluster2 mirror image_33 2024-09-14T08:58:17.738 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:58:17.738 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:17.739 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_33 2024-09-14T08:58:17.739 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_33 2024-09-14T08:58:18.417 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Snapshot ID: 150 2024-09-14T08:58:18.423 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_33 /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_33.status 2024-09-14T08:58:18.423 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:58:18.424 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:18.424 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_33 2024-09-14T08:58:18.424 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_33.status 2024-09-14T08:58:18.424 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_33 --format xml 2024-09-14T08:58:18.424 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:58:18.470 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-14T08:58:18.472 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snapshot_id=150 2024-09-14T08:58:18.472 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:58:18.472 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:58:18.472 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.2 2024-09-14T08:58:18.673 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_33 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_33.status 2024-09-14T08:58:18.673 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:58:18.674 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:18.674 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_33 2024-09-14T08:58:18.674 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_33.status 2024-09-14T08:58:18.674 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_33 --format xml 2024-09-14T08:58:18.674 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:58:18.723 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:58:18.723 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:58:18.725 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:58:18.725 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:58:18.725 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:58:18.725 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:58:18.725 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 150 = '' 2024-09-14T08:58:18.725 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:58:18.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.4 2024-09-14T08:58:19.126 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_33 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_33.status 2024-09-14T08:58:19.127 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:58:19.127 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:19.127 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_33 2024-09-14T08:58:19.127 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_33.status 2024-09-14T08:58:19.127 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_33 --format xml 2024-09-14T08:58:19.127 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:58:19.175 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:58:19.175 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:58:19.177 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:58:19.177 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:58:19.177 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:58:19.177 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:58:19.177 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 150 = '' 2024-09-14T08:58:19.177 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:58:19.177 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.8 2024-09-14T08:58:19.978 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_33 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_33.status 2024-09-14T08:58:19.978 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:58:19.979 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:19.979 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_33 2024-09-14T08:58:19.979 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_33.status 2024-09-14T08:58:19.979 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_33 --format xml 2024-09-14T08:58:19.979 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:58:20.027 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:58:20.029 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=150 2024-09-14T08:58:20.029 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 150 = 150 2024-09-14T08:58:20.029 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:58:20.029 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snap_present cluster1 mirror image_33 snap 2024-09-14T08:58:20.029 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:58:20.029 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:20.029 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_33 2024-09-14T08:58:20.029 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:58:20.029 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:58:20.030 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T08:58:20.030 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:58:21.030 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 info mirror/image_33@snap 2024-09-14T08:58:21.078 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_33': 2024-09-14T08:58:21.078 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T08:58:21.078 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:58:21.078 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 2 2024-09-14T08:58:21.078 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 10bdc6bdb17a 2024-09-14T08:58:21.078 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.10bdc6bdb17a 2024-09-14T08:58:21.079 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:58:21.079 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering, non-primary 2024-09-14T08:58:21.079 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:58:21.079 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:58:21.079 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:50:31 2024 2024-09-14T08:58:21.079 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:50:31 2024 2024-09-14T08:58:21.079 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:50:31 2024 2024-09-14T08:58:21.079 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: protected: False 2024-09-14T08:58:21.079 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:58:21.079 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:58:21.079 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: baaf07f1-8725-4ffd-a1a5-efd3fb31c8b6 2024-09-14T08:58:21.079 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: false 2024-09-14T08:58:21.082 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:58:21.082 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ compare_image_snaps mirror image_33 snap 2024-09-14T08:58:21.082 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:21.082 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_33 2024-09-14T08:58:21.083 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:58:21.083 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local ret=0 2024-09-14T08:58:21.083 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local rmt_export=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_33.export 2024-09-14T08:58:21.083 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local loc_export=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_33.export 2024-09-14T08:58:21.083 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_33.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_33.export 2024-09-14T08:58:21.083 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror export image_33@snap /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_33.export 2024-09-14T08:58:21.312 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T08:58:21.318 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 -p mirror export image_33@snap /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_33.export 2024-09-14T08:58:21.586 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T08:58:21.593 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cmp /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_33.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_33.export 2024-09-14T08:58:21.647 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_33.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_33.export 2024-09-14T08:58:21.678 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:58:21.678 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T08:58:21.679 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_34 2024-09-14T08:58:21.679 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_snap cluster2 mirror image_34 snap 2024-09-14T08:58:21.679 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:58:21.679 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:21.679 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_34 2024-09-14T08:58:21.679 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:58:21.679 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror snap create image_34@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T08:58:22.424 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_started cluster1 mirror image_34 2024-09-14T08:58:22.424 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:58:22.424 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:22.424 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_34 2024-09-14T08:58:22.424 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_state cluster1 mirror image_34 started 2024-09-14T08:58:22.424 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:58:22.424 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:22.424 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_34 2024-09-14T08:58:22.425 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=started 2024-09-14T08:58:22.425 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:58:22.425 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:58:22.425 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:58:23.425 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_replay_state cluster1 mirror image_34 started 2024-09-14T08:58:23.425 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:58:23.425 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:23.425 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_34 2024-09-14T08:58:23.425 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=started 2024-09-14T08:58:23.425 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_result 2024-09-14T08:58:23.425 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=stopped 2024-09-14T08:58:23.427 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_34 2024-09-14T08:58:23.427 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_instance=cluster1 2024-09-14T08:58:23.427 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:58:23.427 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ grep -i state 2024-09-14T08:58:23.427 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T08:58:23.427 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance=cluster1 2024-09-14T08:58:23.428 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local loop_instance 2024-09-14T08:58:23.428 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:58:23.428 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sleep 0 2024-09-14T08:58:23.428 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T08:58:23.428 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ seq 0 3 2024-09-14T08:58:23.431 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:58:23.431 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_34 2024-09-14T08:58:23.431 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:0 2024-09-14T08:58:23.431 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:58:23.431 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:58:23.431 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T08:58:23.431 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:0 2024-09-14T08:58:23.431 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:58:23.431 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:58:23.431 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:58:23.431 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=0 2024-09-14T08:58:23.432 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T08:58:23.432 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:58:23.432 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:58:23.432 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=0 2024-09-14T08:58:23.432 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:58:23.432 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T08:58:23.432 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:0 2024-09-14T08:58:23.432 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:58:23.432 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:58:23.433 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T08:58:23.433 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:0 2024-09-14T08:58:23.433 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:58:23.433 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:58:23.433 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:58:23.433 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:58:23.433 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T08:58:23.433 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:58:23.434 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:58:23.434 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=0 2024-09-14T08:58:23.434 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=0 2024-09-14T08:58:23.434 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T08:58:23.451 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:58:23.451 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:58:23.451 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:58:23.452 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_34 2024-09-14T08:58:23.540 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_34'] 2024-09-14T08:58:23.540 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:58:23.540 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:58:23.543 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:58:23.543 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_34 2024-09-14T08:58:23.543 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:1 2024-09-14T08:58:23.544 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:58:23.544 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:58:23.544 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T08:58:23.544 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:1 2024-09-14T08:58:23.544 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:58:23.544 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:58:23.544 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:58:23.544 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=1 2024-09-14T08:58:23.544 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T08:58:23.544 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:58:23.544 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:58:23.544 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=1 2024-09-14T08:58:23.544 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:58:23.545 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T08:58:23.545 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:1 2024-09-14T08:58:23.545 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:58:23.545 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:58:23.545 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T08:58:23.545 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:1 2024-09-14T08:58:23.545 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:58:23.545 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:58:23.545 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:58:23.545 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:58:23.545 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T08:58:23.545 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:58:23.545 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:58:23.545 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=1 2024-09-14T08:58:23.546 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=1 2024-09-14T08:58:23.546 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T08:58:23.563 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:58:23.563 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:58:23.563 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:58:23.563 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_34 2024-09-14T08:58:23.646 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_34'] 2024-09-14T08:58:23.646 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:58:23.646 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:58:23.649 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:58:23.649 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_34 2024-09-14T08:58:23.649 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:2 2024-09-14T08:58:23.649 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:58:23.649 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:58:23.649 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-14T08:58:23.649 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:2 2024-09-14T08:58:23.649 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:58:23.650 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:58:23.650 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:58:23.650 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=2 2024-09-14T08:58:23.650 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T08:58:23.650 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:58:23.650 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:58:23.650 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=2 2024-09-14T08:58:23.650 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:58:23.650 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-14T08:58:23.650 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:2 2024-09-14T08:58:23.650 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:58:23.650 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:58:23.650 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T08:58:23.651 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:2 2024-09-14T08:58:23.651 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:58:23.651 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:58:23.651 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:58:23.651 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:58:23.651 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-14T08:58:23.651 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:58:23.651 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:58:23.651 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=2 2024-09-14T08:58:23.651 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=2 2024-09-14T08:58:23.651 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T08:58:23.668 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:58:23.668 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:58:23.668 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:58:23.668 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_34 2024-09-14T08:58:23.749 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_34'] 2024-09-14T08:58:23.749 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:58:23.750 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:58:23.753 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:58:23.753 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_34 2024-09-14T08:58:23.753 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:3 2024-09-14T08:58:23.753 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:58:23.753 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:58:23.753 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:3 cluster instance 2024-09-14T08:58:23.753 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:3 2024-09-14T08:58:23.753 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:58:23.753 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:58:23.753 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:58:23.754 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=3 2024-09-14T08:58:23.754 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-14T08:58:23.754 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:58:23.754 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:58:23.754 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=3 2024-09-14T08:58:23.754 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=3 2024-09-14T08:58:23.754 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:3 cluster1 2024-09-14T08:58:23.754 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:3 2024-09-14T08:58:23.754 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:58:23.754 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:58:23.754 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-14T08:58:23.754 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:3 2024-09-14T08:58:23.754 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:58:23.755 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:58:23.755 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:58:23.755 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=3 2024-09-14T08:58:23.755 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 3 = cluster1:3 ']' 2024-09-14T08:58:23.755 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:58:23.755 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:58:23.755 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=3 2024-09-14T08:58:23.755 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=3 2024-09-14T08:58:23.755 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-14T08:58:23.772 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T08:58:23.772 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T08:58:23.772 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T08:58:23.772 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_34 2024-09-14T08:58:23.861 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ return 0 2024-09-14T08:58:23.861 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T08:58:23.862 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ echo ' "state": "Replaying"' 2024-09-14T08:58:23.862 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep -i Replaying 2024-09-14T08:58:23.863 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: "state": "Replaying" 2024-09-14T08:58:23.867 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_state=started 2024-09-14T08:58:23.867 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test started = started 2024-09-14T08:58:23.867 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:58:23.867 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_34 2024-09-14T08:58:23.867 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:58:23.867 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:58:23.867 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:23.867 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_34 2024-09-14T08:58:23.867 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = journal ']' 2024-09-14T08:58:23.867 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = snapshot ']' 2024-09-14T08:58:23.868 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_34 2024-09-14T08:58:23.868 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:58:23.868 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:58:23.868 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:23.868 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_34 2024-09-14T08:58:23.868 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster2-mirror-image_34.status 2024-09-14T08:58:23.868 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster2-mirror-image_34.status 2024-09-14T08:58:23.868 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:58:23.868 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_34.status 2024-09-14T08:58:23.868 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster1-mirror-image_34.status 2024-09-14T08:58:23.868 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster1-mirror-image_34.status 2024-09-14T08:58:23.868 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:58:23.868 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_status_log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_34.status 2024-09-14T08:58:23.868 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ mirror_image_snapshot cluster2 mirror image_34 2024-09-14T08:58:23.869 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:58:23.869 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:23.869 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_34 2024-09-14T08:58:23.869 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_34 2024-09-14T08:58:24.433 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Snapshot ID: 152 2024-09-14T08:58:24.440 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_34 /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_34.status 2024-09-14T08:58:24.440 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:58:24.440 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:24.440 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_34 2024-09-14T08:58:24.440 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_34.status 2024-09-14T08:58:24.440 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_34 --format xml 2024-09-14T08:58:24.440 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:58:24.486 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-14T08:58:24.488 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snapshot_id=152 2024-09-14T08:58:24.488 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:58:24.488 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:58:24.488 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.2 2024-09-14T08:58:24.689 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_34 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_34.status 2024-09-14T08:58:24.690 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:58:24.690 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:24.690 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_34 2024-09-14T08:58:24.690 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_34.status 2024-09-14T08:58:24.690 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_34 --format xml 2024-09-14T08:58:24.690 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:58:24.740 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:58:24.740 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:58:24.741 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:58:24.742 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:58:24.742 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:58:24.742 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:58:24.742 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 152 = '' 2024-09-14T08:58:24.742 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:58:24.742 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.4 2024-09-14T08:58:25.143 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_34 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_34.status 2024-09-14T08:58:25.144 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:58:25.144 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:25.144 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_34 2024-09-14T08:58:25.144 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_34.status 2024-09-14T08:58:25.144 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_34 --format xml 2024-09-14T08:58:25.144 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:58:25.193 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:58:25.193 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:58:25.195 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:58:25.195 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:58:25.195 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:58:25.195 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:58:25.195 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 152 = '' 2024-09-14T08:58:25.196 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:58:25.196 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.8 2024-09-14T08:58:25.997 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_34 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_34.status 2024-09-14T08:58:25.997 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:58:25.997 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:25.997 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_34 2024-09-14T08:58:25.997 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_34.status 2024-09-14T08:58:25.997 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_34 --format xml 2024-09-14T08:58:25.997 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:58:26.044 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:58:26.046 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=152 2024-09-14T08:58:26.047 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 152 = 152 2024-09-14T08:58:26.047 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:58:26.047 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snap_present cluster1 mirror image_34 snap 2024-09-14T08:58:26.047 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:58:26.047 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:26.047 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_34 2024-09-14T08:58:26.047 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:58:26.047 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:58:26.047 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T08:58:26.047 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:58:27.047 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 info mirror/image_34@snap 2024-09-14T08:58:27.090 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_34': 2024-09-14T08:58:27.091 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T08:58:27.091 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:58:27.091 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 2 2024-09-14T08:58:27.091 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 1189a2ecd0cd 2024-09-14T08:58:27.091 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.1189a2ecd0cd 2024-09-14T08:58:27.091 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:58:27.091 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering, non-primary 2024-09-14T08:58:27.091 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:58:27.091 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:58:27.091 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:50:38 2024 2024-09-14T08:58:27.091 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:50:38 2024 2024-09-14T08:58:27.091 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:50:38 2024 2024-09-14T08:58:27.091 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: protected: False 2024-09-14T08:58:27.092 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:58:27.092 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:58:27.092 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: ec5aa9c3-0e81-40f0-a328-f1ef0300304d 2024-09-14T08:58:27.092 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: false 2024-09-14T08:58:27.094 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:58:27.095 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ compare_image_snaps mirror image_34 snap 2024-09-14T08:58:27.095 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:27.095 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_34 2024-09-14T08:58:27.095 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:58:27.095 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local ret=0 2024-09-14T08:58:27.095 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local rmt_export=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_34.export 2024-09-14T08:58:27.095 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local loc_export=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_34.export 2024-09-14T08:58:27.095 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_34.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_34.export 2024-09-14T08:58:27.095 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror export image_34@snap /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_34.export 2024-09-14T08:58:27.295 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T08:58:27.302 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 -p mirror export image_34@snap /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_34.export 2024-09-14T08:58:27.556 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T08:58:27.563 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cmp /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_34.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_34.export 2024-09-14T08:58:27.617 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_34.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_34.export 2024-09-14T08:58:27.648 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:58:27.648 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T08:58:27.648 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_35 2024-09-14T08:58:27.648 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_snap cluster2 mirror image_35 snap 2024-09-14T08:58:27.648 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:58:27.648 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:27.648 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_35 2024-09-14T08:58:27.649 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:58:27.649 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror snap create image_35@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T08:58:28.448 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_started cluster1 mirror image_35 2024-09-14T08:58:28.448 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:58:28.448 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:28.448 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_35 2024-09-14T08:58:28.448 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_state cluster1 mirror image_35 started 2024-09-14T08:58:28.449 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:58:28.449 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:28.449 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_35 2024-09-14T08:58:28.449 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=started 2024-09-14T08:58:28.449 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:58:28.449 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:58:28.449 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:58:29.449 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_replay_state cluster1 mirror image_35 started 2024-09-14T08:58:29.449 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:58:29.450 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:29.450 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_35 2024-09-14T08:58:29.450 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=started 2024-09-14T08:58:29.450 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_result 2024-09-14T08:58:29.450 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=stopped 2024-09-14T08:58:29.451 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_35 2024-09-14T08:58:29.451 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ grep -i state 2024-09-14T08:58:29.451 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_instance=cluster1 2024-09-14T08:58:29.451 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:58:29.451 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T08:58:29.452 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance=cluster1 2024-09-14T08:58:29.452 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local loop_instance 2024-09-14T08:58:29.452 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:58:29.452 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sleep 0 2024-09-14T08:58:29.452 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T08:58:29.455 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ seq 0 3 2024-09-14T08:58:29.455 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:58:29.455 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_35 2024-09-14T08:58:29.455 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:0 2024-09-14T08:58:29.455 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:58:29.456 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:58:29.456 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T08:58:29.456 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:0 2024-09-14T08:58:29.456 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:58:29.456 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:58:29.456 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:58:29.456 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=0 2024-09-14T08:58:29.456 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T08:58:29.456 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:58:29.456 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:58:29.456 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=0 2024-09-14T08:58:29.456 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:58:29.456 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T08:58:29.457 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:0 2024-09-14T08:58:29.457 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:58:29.457 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:58:29.457 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T08:58:29.457 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:0 2024-09-14T08:58:29.457 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:58:29.457 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:58:29.457 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:58:29.457 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:58:29.457 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T08:58:29.457 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:58:29.457 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:58:29.457 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=0 2024-09-14T08:58:29.457 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=0 2024-09-14T08:58:29.458 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T08:58:29.477 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:58:29.477 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:58:29.477 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:58:29.477 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_35 2024-09-14T08:58:29.572 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ return 0 2024-09-14T08:58:29.631 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T08:58:29.631 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ echo ' "state": "Replaying"' 2024-09-14T08:58:29.631 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep -i Replaying 2024-09-14T08:58:29.632 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: "state": "Replaying" 2024-09-14T08:58:29.634 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_state=started 2024-09-14T08:58:29.634 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test started = started 2024-09-14T08:58:29.634 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:58:29.634 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_35 2024-09-14T08:58:29.634 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:58:29.635 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:58:29.635 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:29.635 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_35 2024-09-14T08:58:29.635 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = journal ']' 2024-09-14T08:58:29.635 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = snapshot ']' 2024-09-14T08:58:29.635 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_35 2024-09-14T08:58:29.635 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:58:29.635 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:58:29.635 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:29.635 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_35 2024-09-14T08:58:29.635 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster2-mirror-image_35.status 2024-09-14T08:58:29.635 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster2-mirror-image_35.status 2024-09-14T08:58:29.636 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:58:29.636 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_35.status 2024-09-14T08:58:29.636 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster1-mirror-image_35.status 2024-09-14T08:58:29.636 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster1-mirror-image_35.status 2024-09-14T08:58:29.636 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:58:29.636 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_status_log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_35.status 2024-09-14T08:58:29.636 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ mirror_image_snapshot cluster2 mirror image_35 2024-09-14T08:58:29.636 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:58:29.636 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:29.636 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_35 2024-09-14T08:58:29.636 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_35 2024-09-14T08:58:30.459 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Snapshot ID: 154 2024-09-14T08:58:30.465 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_35 /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_35.status 2024-09-14T08:58:30.465 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:58:30.465 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:30.466 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_35 2024-09-14T08:58:30.466 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_35.status 2024-09-14T08:58:30.466 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_35 --format xml 2024-09-14T08:58:30.466 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:58:30.513 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-14T08:58:30.515 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snapshot_id=154 2024-09-14T08:58:30.515 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:58:30.515 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:58:30.515 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.2 2024-09-14T08:58:30.716 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_35 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_35.status 2024-09-14T08:58:30.716 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:58:30.716 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:30.716 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_35 2024-09-14T08:58:30.716 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_35.status 2024-09-14T08:58:30.716 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_35 --format xml 2024-09-14T08:58:30.716 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:58:30.765 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:58:30.765 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:58:30.767 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:58:30.767 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:58:30.767 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:58:30.767 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:58:30.767 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 154 = '' 2024-09-14T08:58:30.767 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:58:30.767 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.4 2024-09-14T08:58:31.168 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_35 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_35.status 2024-09-14T08:58:31.168 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:58:31.168 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:31.169 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_35 2024-09-14T08:58:31.169 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_35.status 2024-09-14T08:58:31.169 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_35 --format xml 2024-09-14T08:58:31.169 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:58:31.218 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:58:31.218 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:58:31.220 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:58:31.220 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:58:31.220 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:58:31.220 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:58:31.220 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 154 = '' 2024-09-14T08:58:31.221 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:58:31.221 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.8 2024-09-14T08:58:32.021 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_35 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_35.status 2024-09-14T08:58:32.021 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:58:32.022 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:32.022 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_35 2024-09-14T08:58:32.022 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_35.status 2024-09-14T08:58:32.022 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_35 --format xml 2024-09-14T08:58:32.022 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:58:32.072 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:58:32.075 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=154 2024-09-14T08:58:32.075 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 154 = 154 2024-09-14T08:58:32.075 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:58:32.075 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snap_present cluster1 mirror image_35 snap 2024-09-14T08:58:32.076 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:58:32.076 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:32.076 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_35 2024-09-14T08:58:32.076 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:58:32.076 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:58:32.076 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T08:58:32.076 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:58:33.077 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 info mirror/image_35@snap 2024-09-14T08:58:33.123 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_35': 2024-09-14T08:58:33.123 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T08:58:33.123 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:58:33.123 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 2 2024-09-14T08:58:33.123 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 1296604db6e8 2024-09-14T08:58:33.123 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.1296604db6e8 2024-09-14T08:58:33.123 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:58:33.123 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering, non-primary 2024-09-14T08:58:33.123 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:58:33.123 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:58:33.123 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:50:50 2024 2024-09-14T08:58:33.123 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:50:50 2024 2024-09-14T08:58:33.124 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:50:50 2024 2024-09-14T08:58:33.124 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: protected: False 2024-09-14T08:58:33.124 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:58:33.124 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:58:33.124 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: 285cc24e-11c6-46a3-8120-f1a409abec99 2024-09-14T08:58:33.124 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: false 2024-09-14T08:58:33.127 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:58:33.127 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ compare_image_snaps mirror image_35 snap 2024-09-14T08:58:33.127 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:33.127 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_35 2024-09-14T08:58:33.127 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:58:33.127 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local ret=0 2024-09-14T08:58:33.127 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local rmt_export=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_35.export 2024-09-14T08:58:33.127 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local loc_export=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_35.export 2024-09-14T08:58:33.128 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_35.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_35.export 2024-09-14T08:58:33.128 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror export image_35@snap /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_35.export 2024-09-14T08:58:33.331 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T08:58:33.338 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 -p mirror export image_35@snap /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_35.export 2024-09-14T08:58:33.565 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T08:58:33.571 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cmp /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_35.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_35.export 2024-09-14T08:58:33.625 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_35.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_35.export 2024-09-14T08:58:33.657 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:58:33.657 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T08:58:33.657 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_36 2024-09-14T08:58:33.657 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_snap cluster2 mirror image_36 snap 2024-09-14T08:58:33.657 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:58:33.658 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:33.658 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_36 2024-09-14T08:58:33.658 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:58:33.658 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror snap create image_36@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T08:58:34.457 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_started cluster1 mirror image_36 2024-09-14T08:58:34.458 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:58:34.458 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:34.458 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_36 2024-09-14T08:58:34.458 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_state cluster1 mirror image_36 started 2024-09-14T08:58:34.458 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:58:34.458 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:34.458 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_36 2024-09-14T08:58:34.458 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=started 2024-09-14T08:58:34.458 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:58:34.458 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:58:34.458 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:58:35.459 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_replay_state cluster1 mirror image_36 started 2024-09-14T08:58:35.459 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:58:35.459 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:35.460 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_36 2024-09-14T08:58:35.460 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=started 2024-09-14T08:58:35.460 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_result 2024-09-14T08:58:35.460 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=stopped 2024-09-14T08:58:35.460 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_36 2024-09-14T08:58:35.460 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_instance=cluster1 2024-09-14T08:58:35.460 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ grep -i state 2024-09-14T08:58:35.460 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:58:35.460 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T08:58:35.460 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance=cluster1 2024-09-14T08:58:35.461 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local loop_instance 2024-09-14T08:58:35.461 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:58:35.461 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sleep 0 2024-09-14T08:58:35.462 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T08:58:35.462 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ seq 0 3 2024-09-14T08:58:35.463 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:58:35.463 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_36 2024-09-14T08:58:35.463 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:0 2024-09-14T08:58:35.463 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:58:35.463 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:58:35.463 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T08:58:35.463 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:0 2024-09-14T08:58:35.463 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:58:35.463 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:58:35.463 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:58:35.463 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=0 2024-09-14T08:58:35.464 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T08:58:35.464 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:58:35.464 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:58:35.464 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=0 2024-09-14T08:58:35.464 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:58:35.465 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T08:58:35.466 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:0 2024-09-14T08:58:35.466 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:58:35.466 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:58:35.466 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T08:58:35.466 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:0 2024-09-14T08:58:35.466 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:58:35.466 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:58:35.466 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:58:35.466 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:58:35.466 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T08:58:35.466 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:58:35.466 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:58:35.467 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=0 2024-09-14T08:58:35.467 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=0 2024-09-14T08:58:35.467 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T08:58:35.481 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:58:35.481 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:58:35.481 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:58:35.481 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_36 2024-09-14T08:58:35.563 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_36'] 2024-09-14T08:58:35.563 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:58:35.563 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:58:35.566 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:58:35.566 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_36 2024-09-14T08:58:35.566 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:1 2024-09-14T08:58:35.566 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:58:35.566 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:58:35.566 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T08:58:35.566 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:1 2024-09-14T08:58:35.566 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:58:35.567 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:58:35.567 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:58:35.567 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=1 2024-09-14T08:58:35.567 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T08:58:35.567 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:58:35.567 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:58:35.567 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=1 2024-09-14T08:58:35.567 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:58:35.567 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T08:58:35.567 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:1 2024-09-14T08:58:35.567 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:58:35.567 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:58:35.567 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T08:58:35.567 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:1 2024-09-14T08:58:35.567 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:58:35.568 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:58:35.568 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:58:35.568 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:58:35.568 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T08:58:35.568 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:58:35.568 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:58:35.568 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=1 2024-09-14T08:58:35.568 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=1 2024-09-14T08:58:35.568 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T08:58:35.584 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:58:35.584 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:58:35.584 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:58:35.584 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_36 2024-09-14T08:58:35.672 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ return 0 2024-09-14T08:58:35.673 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T08:58:35.673 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ echo ' "state": "Replaying"' 2024-09-14T08:58:35.673 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep -i Replaying 2024-09-14T08:58:35.674 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: "state": "Replaying" 2024-09-14T08:58:35.675 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_state=started 2024-09-14T08:58:35.675 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test started = started 2024-09-14T08:58:35.676 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:58:35.676 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_36 2024-09-14T08:58:35.676 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:58:35.676 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:58:35.676 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:35.676 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_36 2024-09-14T08:58:35.676 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = journal ']' 2024-09-14T08:58:35.676 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = snapshot ']' 2024-09-14T08:58:35.676 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_36 2024-09-14T08:58:35.677 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:58:35.677 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:58:35.677 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:35.677 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_36 2024-09-14T08:58:35.677 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster2-mirror-image_36.status 2024-09-14T08:58:35.677 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster2-mirror-image_36.status 2024-09-14T08:58:35.677 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:58:35.678 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_36.status 2024-09-14T08:58:35.678 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster1-mirror-image_36.status 2024-09-14T08:58:35.678 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster1-mirror-image_36.status 2024-09-14T08:58:35.678 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:58:35.678 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_status_log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_36.status 2024-09-14T08:58:35.678 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ mirror_image_snapshot cluster2 mirror image_36 2024-09-14T08:58:35.679 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:58:35.679 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:35.679 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_36 2024-09-14T08:58:35.679 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_36 2024-09-14T08:58:36.471 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Snapshot ID: 156 2024-09-14T08:58:36.477 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_36 /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_36.status 2024-09-14T08:58:36.477 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:58:36.477 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:36.477 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_36 2024-09-14T08:58:36.477 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_36.status 2024-09-14T08:58:36.477 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_36 --format xml 2024-09-14T08:58:36.477 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:58:36.523 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-14T08:58:36.524 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snapshot_id=156 2024-09-14T08:58:36.525 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:58:36.525 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:58:36.525 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.2 2024-09-14T08:58:36.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_36 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_36.status 2024-09-14T08:58:36.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:58:36.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:36.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_36 2024-09-14T08:58:36.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_36.status 2024-09-14T08:58:36.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_36 --format xml 2024-09-14T08:58:36.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:58:36.774 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:58:36.776 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=70 2024-09-14T08:58:36.776 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 156 = 70 2024-09-14T08:58:36.776 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:58:36.776 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.4 2024-09-14T08:58:37.177 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_36 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_36.status 2024-09-14T08:58:37.178 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:58:37.178 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:37.178 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_36 2024-09-14T08:58:37.178 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_36.status 2024-09-14T08:58:37.178 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_36 --format xml 2024-09-14T08:58:37.178 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:58:37.228 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:58:37.230 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=70 2024-09-14T08:58:37.230 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 156 = 70 2024-09-14T08:58:37.230 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:58:37.230 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.8 2024-09-14T08:58:38.032 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_36 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_36.status 2024-09-14T08:58:38.032 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:58:38.032 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:38.032 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_36 2024-09-14T08:58:38.032 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_36.status 2024-09-14T08:58:38.032 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_36 --format xml 2024-09-14T08:58:38.032 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:58:38.086 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:58:38.086 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:58:38.088 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:58:38.088 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:58:38.088 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:58:38.088 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:58:38.088 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 156 = '' 2024-09-14T08:58:38.088 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:58:38.089 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1.6 2024-09-14T08:58:39.690 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_36 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_36.status 2024-09-14T08:58:39.690 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:58:39.690 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:39.690 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_36 2024-09-14T08:58:39.690 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_36.status 2024-09-14T08:58:39.690 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_36 --format xml 2024-09-14T08:58:39.690 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:58:39.739 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:58:39.741 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=156 2024-09-14T08:58:39.741 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 156 = 156 2024-09-14T08:58:39.741 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:58:39.741 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snap_present cluster1 mirror image_36 snap 2024-09-14T08:58:39.742 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:58:39.742 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:39.742 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_36 2024-09-14T08:58:39.742 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:58:39.742 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:58:39.742 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T08:58:39.742 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:58:40.742 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 info mirror/image_36@snap 2024-09-14T08:58:40.792 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_36': 2024-09-14T08:58:40.792 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T08:58:40.792 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:58:40.792 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 2 2024-09-14T08:58:40.792 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 12a066e0450f 2024-09-14T08:58:40.792 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.12a066e0450f 2024-09-14T08:58:40.792 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:58:40.792 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering, non-primary 2024-09-14T08:58:40.792 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:58:40.793 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:58:40.793 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:51:01 2024 2024-09-14T08:58:40.793 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:51:01 2024 2024-09-14T08:58:40.793 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:51:01 2024 2024-09-14T08:58:40.793 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: protected: False 2024-09-14T08:58:40.793 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:58:40.793 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:58:40.793 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: 1e059464-8982-44b9-a52b-107ceeccb93e 2024-09-14T08:58:40.793 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: false 2024-09-14T08:58:40.796 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:58:40.796 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ compare_image_snaps mirror image_36 snap 2024-09-14T08:58:40.796 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:40.796 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_36 2024-09-14T08:58:40.796 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:58:40.797 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local ret=0 2024-09-14T08:58:40.797 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local rmt_export=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_36.export 2024-09-14T08:58:40.797 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local loc_export=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_36.export 2024-09-14T08:58:40.797 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_36.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_36.export 2024-09-14T08:58:40.797 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror export image_36@snap /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_36.export 2024-09-14T08:58:40.994 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T08:58:41.001 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 -p mirror export image_36@snap /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_36.export 2024-09-14T08:58:41.253 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T08:58:41.261 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cmp /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_36.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_36.export 2024-09-14T08:58:41.316 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_36.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_36.export 2024-09-14T08:58:41.349 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:58:41.349 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T08:58:41.349 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_37 2024-09-14T08:58:41.349 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_snap cluster2 mirror image_37 snap 2024-09-14T08:58:41.349 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:58:41.349 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:41.349 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_37 2024-09-14T08:58:41.349 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:58:41.349 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror snap create image_37@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T08:58:41.571 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_started cluster1 mirror image_37 2024-09-14T08:58:41.572 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:58:41.572 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:41.572 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_37 2024-09-14T08:58:41.572 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_state cluster1 mirror image_37 started 2024-09-14T08:58:41.572 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:58:41.572 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:41.572 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_37 2024-09-14T08:58:41.572 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=started 2024-09-14T08:58:41.572 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:58:41.572 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:58:41.572 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:58:42.574 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_replay_state cluster1 mirror image_37 started 2024-09-14T08:58:42.574 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:58:42.574 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:42.574 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_37 2024-09-14T08:58:42.574 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=started 2024-09-14T08:58:42.574 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_result 2024-09-14T08:58:42.574 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=stopped 2024-09-14T08:58:42.574 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_37 2024-09-14T08:58:42.574 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_instance=cluster1 2024-09-14T08:58:42.574 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ grep -i state 2024-09-14T08:58:42.574 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:58:42.575 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T08:58:42.575 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance=cluster1 2024-09-14T08:58:42.575 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local loop_instance 2024-09-14T08:58:42.575 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:58:42.575 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sleep 0 2024-09-14T08:58:42.578 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T08:58:42.579 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ seq 0 3 2024-09-14T08:58:42.579 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:58:42.579 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_37 2024-09-14T08:58:42.579 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:0 2024-09-14T08:58:42.579 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:58:42.579 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:58:42.580 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T08:58:42.580 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:0 2024-09-14T08:58:42.580 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:58:42.580 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:58:42.580 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:58:42.580 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=0 2024-09-14T08:58:42.580 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T08:58:42.580 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:58:42.580 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:58:42.580 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=0 2024-09-14T08:58:42.581 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:58:42.581 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T08:58:42.581 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:0 2024-09-14T08:58:42.581 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:58:42.581 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:58:42.581 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T08:58:42.581 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:0 2024-09-14T08:58:42.581 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:58:42.581 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:58:42.581 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:58:42.581 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:58:42.581 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T08:58:42.582 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:58:42.582 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:58:42.582 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=0 2024-09-14T08:58:42.582 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=0 2024-09-14T08:58:42.582 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T08:58:42.596 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:58:42.596 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:58:42.596 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:58:42.596 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_37 2024-09-14T08:58:42.681 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_37'] 2024-09-14T08:58:42.681 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:58:42.681 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:58:42.681 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:58:42.681 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_37 2024-09-14T08:58:42.681 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:1 2024-09-14T08:58:42.681 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:58:42.681 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:58:42.681 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T08:58:42.681 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:1 2024-09-14T08:58:42.682 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:58:42.682 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:58:42.682 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:58:42.682 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=1 2024-09-14T08:58:42.682 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T08:58:42.682 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:58:42.682 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:58:42.682 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=1 2024-09-14T08:58:42.682 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:58:42.682 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T08:58:42.682 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:1 2024-09-14T08:58:42.683 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:58:42.683 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:58:42.683 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T08:58:42.683 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:1 2024-09-14T08:58:42.683 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:58:42.683 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:58:42.683 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:58:42.683 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:58:42.683 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T08:58:42.683 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:58:42.683 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:58:42.683 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=1 2024-09-14T08:58:42.683 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=1 2024-09-14T08:58:42.683 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T08:58:42.699 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:58:42.699 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:58:42.699 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:58:42.699 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_37 2024-09-14T08:58:42.783 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_37'] 2024-09-14T08:58:42.783 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:58:42.783 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:58:42.787 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:58:42.787 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_37 2024-09-14T08:58:42.787 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:2 2024-09-14T08:58:42.787 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:58:42.787 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:58:42.787 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-14T08:58:42.787 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:2 2024-09-14T08:58:42.787 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:58:42.787 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:58:42.787 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:58:42.788 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=2 2024-09-14T08:58:42.788 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T08:58:42.788 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:58:42.788 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:58:42.788 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=2 2024-09-14T08:58:42.788 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:58:42.788 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-14T08:58:42.788 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:2 2024-09-14T08:58:42.788 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:58:42.788 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:58:42.788 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T08:58:42.788 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:2 2024-09-14T08:58:42.788 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:58:42.789 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:58:42.789 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:58:42.789 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:58:42.789 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-14T08:58:42.789 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:58:42.789 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:58:42.789 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=2 2024-09-14T08:58:42.789 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=2 2024-09-14T08:58:42.789 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T08:58:42.804 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:58:42.805 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:58:42.805 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:58:42.805 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_37 2024-09-14T08:58:42.894 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ return 0 2024-09-14T08:58:42.895 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T08:58:42.895 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ echo ' "state": "Replaying"' 2024-09-14T08:58:42.895 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep -i Replaying 2024-09-14T08:58:42.896 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: "state": "Replaying" 2024-09-14T08:58:42.897 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_state=started 2024-09-14T08:58:42.897 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test started = started 2024-09-14T08:58:42.898 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:58:42.898 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_37 2024-09-14T08:58:42.898 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:58:42.898 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:58:42.898 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:42.898 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_37 2024-09-14T08:58:42.898 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = journal ']' 2024-09-14T08:58:42.898 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = snapshot ']' 2024-09-14T08:58:42.898 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_37 2024-09-14T08:58:42.898 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:58:42.898 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:58:42.898 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:42.898 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_37 2024-09-14T08:58:42.898 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster2-mirror-image_37.status 2024-09-14T08:58:42.899 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster2-mirror-image_37.status 2024-09-14T08:58:42.899 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:58:42.900 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_37.status 2024-09-14T08:58:42.900 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster1-mirror-image_37.status 2024-09-14T08:58:42.900 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster1-mirror-image_37.status 2024-09-14T08:58:42.900 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:58:42.900 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_status_log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_37.status 2024-09-14T08:58:42.900 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ mirror_image_snapshot cluster2 mirror image_37 2024-09-14T08:58:42.900 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:58:42.900 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:42.900 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_37 2024-09-14T08:58:42.900 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_37 2024-09-14T08:58:43.581 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Snapshot ID: 158 2024-09-14T08:58:43.588 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_37 /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_37.status 2024-09-14T08:58:43.588 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:58:43.588 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:43.588 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_37 2024-09-14T08:58:43.588 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_37.status 2024-09-14T08:58:43.588 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_37 --format xml 2024-09-14T08:58:43.588 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:58:43.634 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-14T08:58:43.636 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snapshot_id=158 2024-09-14T08:58:43.636 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:58:43.636 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:58:43.636 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.2 2024-09-14T08:58:43.837 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_37 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_37.status 2024-09-14T08:58:43.838 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:58:43.838 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:43.838 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_37 2024-09-14T08:58:43.838 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_37.status 2024-09-14T08:58:43.838 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_37 --format xml 2024-09-14T08:58:43.838 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:58:44.089 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:58:44.091 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=71 2024-09-14T08:58:44.091 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 158 = 71 2024-09-14T08:58:44.091 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:58:44.091 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.4 2024-09-14T08:58:44.492 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_37 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_37.status 2024-09-14T08:58:44.492 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:58:44.493 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:44.493 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_37 2024-09-14T08:58:44.493 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_37.status 2024-09-14T08:58:44.493 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_37 --format xml 2024-09-14T08:58:44.493 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:58:44.541 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:58:44.541 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:58:44.542 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:58:44.542 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:58:44.542 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:58:44.543 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:58:44.543 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 158 = '' 2024-09-14T08:58:44.543 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:58:44.543 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.8 2024-09-14T08:58:45.164 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.7203119651065142) > weight (0.25) 2024-09-14T08:58:45.165 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.1 2024-09-14T08:58:45.165 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Sent signal 15 2024-09-14T08:58:45.165 INFO:tasks.rbd_mirror_thrash:waiting for 1 secs before reviving daemons 2024-09-14T08:58:45.166 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi164.stderr:2024-09-14T08:58:45.166+0000 7feda92b4640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 (PID: 47049) UID: 1000 2024-09-14T08:58:45.344 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_37 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_37.status 2024-09-14T08:58:45.345 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:58:45.345 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:45.345 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_37 2024-09-14T08:58:45.345 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_37.status 2024-09-14T08:58:45.345 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_37 --format xml 2024-09-14T08:58:45.345 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:58:45.395 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:58:45.395 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:58:45.396 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:58:45.397 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:58:45.397 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:58:45.397 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:58:45.397 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 158 = '' 2024-09-14T08:58:45.397 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:58:45.397 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1.6 2024-09-14T08:58:46.166 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.1 2024-09-14T08:58:46.166 INFO:teuthology.orchestra.run:waiting for 600 2024-09-14T08:58:46.167 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.1 2024-09-14T08:58:46.167 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Restarting daemon 2024-09-14T08:58:46.167 DEBUG:teuthology.orchestra.run.smithi164:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 2024-09-14T08:58:46.170 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Started 2024-09-14T08:58:46.170 INFO:tasks.rbd_mirror_thrash:waiting for 100 secs before thrashing 2024-09-14T08:58:46.998 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_37 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_37.status 2024-09-14T08:58:46.998 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:58:46.998 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:46.999 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_37 2024-09-14T08:58:46.999 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_37.status 2024-09-14T08:58:46.999 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_37 --format xml 2024-09-14T08:58:46.999 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:58:47.046 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:58:47.048 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=158 2024-09-14T08:58:47.048 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 158 = 158 2024-09-14T08:58:47.048 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:58:47.048 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snap_present cluster1 mirror image_37 snap 2024-09-14T08:58:47.048 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:58:47.048 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:47.049 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_37 2024-09-14T08:58:47.049 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:58:47.049 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:58:47.049 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T08:58:47.049 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:58:48.049 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 info mirror/image_37@snap 2024-09-14T08:58:48.096 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_37': 2024-09-14T08:58:48.096 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T08:58:48.096 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:58:48.096 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 2 2024-09-14T08:58:48.096 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 10bd4c774daa 2024-09-14T08:58:48.096 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.10bd4c774daa 2024-09-14T08:58:48.096 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:58:48.097 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering, non-primary 2024-09-14T08:58:48.097 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:58:48.097 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:58:48.097 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:51:10 2024 2024-09-14T08:58:48.097 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:51:10 2024 2024-09-14T08:58:48.097 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:51:10 2024 2024-09-14T08:58:48.097 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: protected: False 2024-09-14T08:58:48.097 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:58:48.097 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:58:48.097 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: f4bb57da-c08d-42af-8cd0-9868b7375dfb 2024-09-14T08:58:48.097 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: false 2024-09-14T08:58:48.100 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:58:48.101 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ compare_image_snaps mirror image_37 snap 2024-09-14T08:58:48.101 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:48.101 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_37 2024-09-14T08:58:48.101 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:58:48.101 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local ret=0 2024-09-14T08:58:48.101 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local rmt_export=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_37.export 2024-09-14T08:58:48.101 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local loc_export=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_37.export 2024-09-14T08:58:48.101 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_37.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_37.export 2024-09-14T08:58:48.101 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror export image_37@snap /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_37.export 2024-09-14T08:58:48.284 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T08:58:48.292 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 -p mirror export image_37@snap /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_37.export 2024-09-14T08:58:48.552 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T08:58:48.559 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cmp /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_37.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_37.export 2024-09-14T08:58:48.615 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_37.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_37.export 2024-09-14T08:58:48.647 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:58:48.647 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T08:58:48.647 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_38 2024-09-14T08:58:48.647 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_snap cluster2 mirror image_38 snap 2024-09-14T08:58:48.647 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:58:48.647 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:48.647 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_38 2024-09-14T08:58:48.647 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:58:48.647 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror snap create image_38@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T08:58:49.583 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_started cluster1 mirror image_38 2024-09-14T08:58:49.584 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:58:49.584 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:49.584 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_38 2024-09-14T08:58:49.584 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_state cluster1 mirror image_38 started 2024-09-14T08:58:49.584 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:58:49.584 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:49.584 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_38 2024-09-14T08:58:49.584 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=started 2024-09-14T08:58:49.584 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:58:49.584 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:58:49.584 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:58:50.585 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_replay_state cluster1 mirror image_38 started 2024-09-14T08:58:50.585 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:58:50.585 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:50.585 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_38 2024-09-14T08:58:50.585 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=started 2024-09-14T08:58:50.585 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_result 2024-09-14T08:58:50.586 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=stopped 2024-09-14T08:58:50.586 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_38 2024-09-14T08:58:50.586 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_instance=cluster1 2024-09-14T08:58:50.586 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ grep -i state 2024-09-14T08:58:50.586 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:58:50.586 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T08:58:50.586 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance=cluster1 2024-09-14T08:58:50.586 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local loop_instance 2024-09-14T08:58:50.586 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:58:50.586 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sleep 0 2024-09-14T08:58:50.588 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T08:58:50.589 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ seq 0 3 2024-09-14T08:58:50.589 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:58:50.589 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_38 2024-09-14T08:58:50.589 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:0 2024-09-14T08:58:50.589 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:58:50.589 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:58:50.589 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T08:58:50.589 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:0 2024-09-14T08:58:50.589 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:58:50.589 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:58:50.589 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:58:50.589 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=0 2024-09-14T08:58:50.589 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T08:58:50.589 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:58:50.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:58:50.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=0 2024-09-14T08:58:50.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:58:50.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T08:58:50.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:0 2024-09-14T08:58:50.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:58:50.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:58:50.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T08:58:50.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:0 2024-09-14T08:58:50.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:58:50.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:58:50.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:58:50.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:58:50.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T08:58:50.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:58:50.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:58:50.591 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=0 2024-09-14T08:58:50.591 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=0 2024-09-14T08:58:50.591 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T08:58:50.611 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:58:50.611 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:58:50.611 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:58:50.611 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_38 2024-09-14T08:58:50.697 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_38'] 2024-09-14T08:58:50.698 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:58:50.698 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:58:50.701 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:58:50.701 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_38 2024-09-14T08:58:50.701 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:1 2024-09-14T08:58:50.701 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:58:50.701 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:58:50.701 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T08:58:50.701 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:1 2024-09-14T08:58:50.701 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:58:50.701 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:58:50.702 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:58:50.702 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=1 2024-09-14T08:58:50.702 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T08:58:50.702 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:58:50.702 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:58:50.702 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=1 2024-09-14T08:58:50.702 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:58:50.702 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T08:58:50.702 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:1 2024-09-14T08:58:50.702 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:58:50.702 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:58:50.702 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T08:58:50.703 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:1 2024-09-14T08:58:50.703 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:58:50.703 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:58:50.703 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:58:50.703 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:58:50.703 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T08:58:50.703 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:58:50.703 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:58:50.703 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=1 2024-09-14T08:58:50.703 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=1 2024-09-14T08:58:50.703 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T08:58:50.720 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:58:50.720 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:58:50.720 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:58:50.720 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_38 2024-09-14T08:58:50.794 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_38'] 2024-09-14T08:58:50.794 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:58:50.794 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:58:50.797 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:58:50.797 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_38 2024-09-14T08:58:50.797 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:2 2024-09-14T08:58:50.797 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:58:50.797 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:58:50.797 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-14T08:58:50.797 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:2 2024-09-14T08:58:50.797 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:58:50.797 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:58:50.797 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:58:50.797 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=2 2024-09-14T08:58:50.797 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T08:58:50.798 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:58:50.798 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:58:50.798 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=2 2024-09-14T08:58:50.798 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:58:50.798 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-14T08:58:50.798 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:2 2024-09-14T08:58:50.798 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:58:50.798 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:58:50.798 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T08:58:50.799 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:2 2024-09-14T08:58:50.799 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:58:50.799 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:58:50.799 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:58:50.799 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:58:50.799 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-14T08:58:50.799 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:58:50.799 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:58:50.799 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=2 2024-09-14T08:58:50.799 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=2 2024-09-14T08:58:50.799 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T08:58:50.815 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:58:50.815 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:58:50.816 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:58:50.816 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_38 2024-09-14T08:58:50.898 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_38'] 2024-09-14T08:58:50.898 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:58:50.898 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:58:50.901 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:58:50.901 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_38 2024-09-14T08:58:50.901 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:3 2024-09-14T08:58:50.901 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:58:50.901 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:58:50.902 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:3 cluster instance 2024-09-14T08:58:50.902 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:3 2024-09-14T08:58:50.902 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:58:50.902 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:58:50.902 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:58:50.902 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=3 2024-09-14T08:58:50.903 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-14T08:58:50.903 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:58:50.903 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:58:50.903 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=3 2024-09-14T08:58:50.903 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=3 2024-09-14T08:58:50.903 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:3 cluster1 2024-09-14T08:58:50.903 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:3 2024-09-14T08:58:50.904 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:58:50.904 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:58:50.904 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-14T08:58:50.904 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:3 2024-09-14T08:58:50.904 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:58:50.904 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:58:50.905 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:58:50.905 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=3 2024-09-14T08:58:50.905 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 3 = cluster1:3 ']' 2024-09-14T08:58:50.905 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:58:50.905 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:58:50.905 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=3 2024-09-14T08:58:50.906 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=3 2024-09-14T08:58:50.906 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-14T08:58:50.919 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T08:58:50.919 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T08:58:50.919 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T08:58:50.919 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_38 2024-09-14T08:58:51.008 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ return 0 2024-09-14T08:58:51.009 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T08:58:51.009 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ echo ' "state": "Replaying"' 2024-09-14T08:58:51.009 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep -i Replaying 2024-09-14T08:58:51.010 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: "state": "Replaying" 2024-09-14T08:58:51.012 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_state=started 2024-09-14T08:58:51.012 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test started = started 2024-09-14T08:58:51.012 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:58:51.012 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_38 2024-09-14T08:58:51.012 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:58:51.012 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:58:51.012 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:51.012 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_38 2024-09-14T08:58:51.012 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = journal ']' 2024-09-14T08:58:51.012 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = snapshot ']' 2024-09-14T08:58:51.012 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_38 2024-09-14T08:58:51.012 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:58:51.013 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:58:51.013 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:51.013 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_38 2024-09-14T08:58:51.013 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster2-mirror-image_38.status 2024-09-14T08:58:51.013 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster2-mirror-image_38.status 2024-09-14T08:58:51.013 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:58:51.013 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_38.status 2024-09-14T08:58:51.014 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster1-mirror-image_38.status 2024-09-14T08:58:51.014 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster1-mirror-image_38.status 2024-09-14T08:58:51.014 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:58:51.014 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_status_log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_38.status 2024-09-14T08:58:51.014 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ mirror_image_snapshot cluster2 mirror image_38 2024-09-14T08:58:51.014 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:58:51.015 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:51.015 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_38 2024-09-14T08:58:51.015 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_38 2024-09-14T08:58:51.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Snapshot ID: 160 2024-09-14T08:58:51.596 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_38 /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_38.status 2024-09-14T08:58:51.596 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:58:51.596 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:51.596 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_38 2024-09-14T08:58:51.596 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_38.status 2024-09-14T08:58:51.597 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_38 --format xml 2024-09-14T08:58:51.597 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:58:51.648 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-14T08:58:51.650 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snapshot_id=160 2024-09-14T08:58:51.650 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:58:51.650 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:58:51.651 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.2 2024-09-14T08:58:51.852 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_38 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_38.status 2024-09-14T08:58:51.852 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:58:51.852 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:51.852 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_38 2024-09-14T08:58:51.852 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_38.status 2024-09-14T08:58:51.852 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_38 --format xml 2024-09-14T08:58:51.853 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:58:51.900 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:58:51.902 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=72 2024-09-14T08:58:51.902 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 160 = 72 2024-09-14T08:58:51.902 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:58:51.902 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.4 2024-09-14T08:58:52.303 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_38 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_38.status 2024-09-14T08:58:52.303 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:58:52.303 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:52.304 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_38 2024-09-14T08:58:52.304 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_38.status 2024-09-14T08:58:52.304 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_38 --format xml 2024-09-14T08:58:52.304 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:58:52.352 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:58:52.354 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=72 2024-09-14T08:58:52.354 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 160 = 72 2024-09-14T08:58:52.354 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:58:52.354 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.8 2024-09-14T08:58:53.155 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_38 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_38.status 2024-09-14T08:58:53.156 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:58:53.156 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:53.156 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_38 2024-09-14T08:58:53.156 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_38.status 2024-09-14T08:58:53.156 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_38 --format xml 2024-09-14T08:58:53.156 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:58:53.209 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:58:53.210 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:58:53.211 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:58:53.211 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:58:53.211 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:58:53.212 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:58:53.212 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 160 = '' 2024-09-14T08:58:53.212 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:58:53.212 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1.6 2024-09-14T08:58:54.813 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_38 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_38.status 2024-09-14T08:58:54.814 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:58:54.814 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:54.814 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_38 2024-09-14T08:58:54.814 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_38.status 2024-09-14T08:58:54.814 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_38 --format xml 2024-09-14T08:58:54.814 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:58:54.870 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:58:54.870 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:58:54.871 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:58:54.871 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:58:54.871 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:58:54.872 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:58:54.872 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 160 = '' 2024-09-14T08:58:54.872 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:58:54.872 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 2 2024-09-14T08:58:56.873 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_38 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_38.status 2024-09-14T08:58:56.874 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:58:56.874 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:56.874 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_38 2024-09-14T08:58:56.874 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_38.status 2024-09-14T08:58:56.874 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_38 --format xml 2024-09-14T08:58:56.874 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:58:56.928 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:58:56.930 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=72 2024-09-14T08:58:56.930 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 160 = 72 2024-09-14T08:58:56.930 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:58:56.930 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 2 2024-09-14T08:58:58.931 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_38 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_38.status 2024-09-14T08:58:58.931 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:58:58.931 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:58.931 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_38 2024-09-14T08:58:58.932 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_38.status 2024-09-14T08:58:58.932 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_38 --format xml 2024-09-14T08:58:58.932 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:58:58.985 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:58:58.988 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=160 2024-09-14T08:58:58.988 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 160 = 160 2024-09-14T08:58:58.988 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:58:58.988 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snap_present cluster1 mirror image_38 snap 2024-09-14T08:58:58.988 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:58:58.988 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:58:58.988 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_38 2024-09-14T08:58:58.988 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:58:58.988 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:58:58.988 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T08:58:58.988 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:58:59.988 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 info mirror/image_38@snap 2024-09-14T08:59:00.041 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_38': 2024-09-14T08:59:00.041 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T08:59:00.041 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:59:00.041 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 2 2024-09-14T08:59:00.041 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 11894ec4469a 2024-09-14T08:59:00.042 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.11894ec4469a 2024-09-14T08:59:00.042 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:59:00.042 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering, non-primary 2024-09-14T08:59:00.042 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:59:00.042 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:59:00.042 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:51:17 2024 2024-09-14T08:59:00.042 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:51:17 2024 2024-09-14T08:59:00.042 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:51:17 2024 2024-09-14T08:59:00.042 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: protected: False 2024-09-14T08:59:00.042 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:59:00.042 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:59:00.042 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: 56c69995-3ca1-4975-bc59-f14d91322ad4 2024-09-14T08:59:00.042 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: false 2024-09-14T08:59:00.045 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:59:00.045 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ compare_image_snaps mirror image_38 snap 2024-09-14T08:59:00.045 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:59:00.045 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_38 2024-09-14T08:59:00.045 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:59:00.045 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local ret=0 2024-09-14T08:59:00.046 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local rmt_export=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_38.export 2024-09-14T08:59:00.046 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local loc_export=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_38.export 2024-09-14T08:59:00.046 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_38.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_38.export 2024-09-14T08:59:00.046 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror export image_38@snap /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_38.export 2024-09-14T08:59:00.257 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T08:59:00.264 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 -p mirror export image_38@snap /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_38.export 2024-09-14T08:59:00.487 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T08:59:00.493 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cmp /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_38.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_38.export 2024-09-14T08:59:00.548 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_38.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_38.export 2024-09-14T08:59:00.580 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:59:00.580 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T08:59:00.580 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_39 2024-09-14T08:59:00.580 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_snap cluster2 mirror image_39 snap 2024-09-14T08:59:00.580 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:59:00.580 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:59:00.580 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_39 2024-09-14T08:59:00.580 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:59:00.580 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror snap create image_39@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T08:59:01.602 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_started cluster1 mirror image_39 2024-09-14T08:59:01.602 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:59:01.602 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:59:01.603 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_39 2024-09-14T08:59:01.603 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_state cluster1 mirror image_39 started 2024-09-14T08:59:01.603 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:59:01.603 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:59:01.603 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_39 2024-09-14T08:59:01.603 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=started 2024-09-14T08:59:01.603 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:59:01.603 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:59:01.603 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:59:02.603 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_replay_state cluster1 mirror image_39 started 2024-09-14T08:59:02.604 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:59:02.604 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:59:02.604 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_39 2024-09-14T08:59:02.604 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=started 2024-09-14T08:59:02.604 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_result 2024-09-14T08:59:02.604 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=stopped 2024-09-14T08:59:02.605 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_39 2024-09-14T08:59:02.605 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_instance=cluster1 2024-09-14T08:59:02.605 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ grep -i state 2024-09-14T08:59:02.605 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:59:02.605 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T08:59:02.605 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance=cluster1 2024-09-14T08:59:02.605 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local loop_instance 2024-09-14T08:59:02.605 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:59:02.605 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sleep 0 2024-09-14T08:59:02.606 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T08:59:02.608 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ seq 0 3 2024-09-14T08:59:02.608 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:59:02.608 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_39 2024-09-14T08:59:02.608 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:0 2024-09-14T08:59:02.608 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:59:02.608 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:59:02.609 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T08:59:02.609 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:0 2024-09-14T08:59:02.609 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:59:02.609 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:59:02.609 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:59:02.609 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=0 2024-09-14T08:59:02.609 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T08:59:02.609 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:59:02.609 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:59:02.609 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=0 2024-09-14T08:59:02.609 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:59:02.609 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T08:59:02.609 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:0 2024-09-14T08:59:02.609 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:59:02.609 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:59:02.610 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T08:59:02.610 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:0 2024-09-14T08:59:02.610 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:59:02.610 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:59:02.610 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:59:02.610 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:59:02.610 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T08:59:02.610 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:59:02.610 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:59:02.610 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=0 2024-09-14T08:59:02.610 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=0 2024-09-14T08:59:02.610 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T08:59:02.630 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:59:02.630 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:59:02.630 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:59:02.630 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_39 2024-09-14T08:59:02.719 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_39'] 2024-09-14T08:59:02.720 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:59:02.720 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:59:02.723 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:59:02.723 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_39 2024-09-14T08:59:02.723 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:1 2024-09-14T08:59:02.723 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:59:02.723 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:59:02.723 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T08:59:02.723 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:1 2024-09-14T08:59:02.723 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:59:02.723 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:59:02.723 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:59:02.723 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=1 2024-09-14T08:59:02.723 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T08:59:02.723 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:59:02.723 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:59:02.724 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=1 2024-09-14T08:59:02.724 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:59:02.724 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T08:59:02.724 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:1 2024-09-14T08:59:02.724 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:59:02.724 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:59:02.724 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T08:59:02.724 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:1 2024-09-14T08:59:02.724 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:59:02.724 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:59:02.724 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:59:02.724 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:59:02.724 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T08:59:02.724 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:59:02.725 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:59:02.725 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=1 2024-09-14T08:59:02.725 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=1 2024-09-14T08:59:02.725 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T08:59:02.742 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:59:02.742 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:59:02.742 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:59:02.742 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_39 2024-09-14T08:59:02.820 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_39'] 2024-09-14T08:59:02.820 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:59:02.820 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:59:02.823 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:59:02.823 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_39 2024-09-14T08:59:02.823 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:2 2024-09-14T08:59:02.823 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:59:02.823 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:59:02.823 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-14T08:59:02.823 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:2 2024-09-14T08:59:02.824 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:59:02.824 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:59:02.824 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:59:02.824 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=2 2024-09-14T08:59:02.824 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T08:59:02.824 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:59:02.824 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:59:02.824 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=2 2024-09-14T08:59:02.824 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:59:02.824 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-14T08:59:02.824 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:2 2024-09-14T08:59:02.824 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:59:02.824 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:59:02.825 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T08:59:02.825 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:2 2024-09-14T08:59:02.825 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:59:02.825 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:59:02.825 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:59:02.825 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:59:02.825 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-14T08:59:02.825 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:59:02.825 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:59:02.825 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=2 2024-09-14T08:59:02.825 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=2 2024-09-14T08:59:02.825 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T08:59:02.841 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:59:02.841 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:59:02.841 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:59:02.841 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_39 2024-09-14T08:59:02.921 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_39'] 2024-09-14T08:59:02.921 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:59:02.921 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:59:02.924 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:59:02.924 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_39 2024-09-14T08:59:02.924 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:3 2024-09-14T08:59:02.924 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:59:02.925 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:59:02.925 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:3 cluster instance 2024-09-14T08:59:02.925 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:3 2024-09-14T08:59:02.925 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:59:02.925 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:59:02.925 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:59:02.925 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=3 2024-09-14T08:59:02.925 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-14T08:59:02.925 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:59:02.925 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:59:02.925 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=3 2024-09-14T08:59:02.925 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=3 2024-09-14T08:59:02.925 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:3 cluster1 2024-09-14T08:59:02.925 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:3 2024-09-14T08:59:02.926 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:59:02.926 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:59:02.926 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-14T08:59:02.926 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:3 2024-09-14T08:59:02.926 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:59:02.926 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:59:02.926 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:59:02.926 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=3 2024-09-14T08:59:02.926 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 3 = cluster1:3 ']' 2024-09-14T08:59:02.926 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:59:02.926 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:59:02.926 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=3 2024-09-14T08:59:02.926 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=3 2024-09-14T08:59:02.926 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-14T08:59:02.942 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T08:59:02.942 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T08:59:02.943 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T08:59:02.943 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_39 2024-09-14T08:59:03.021 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_39'] 2024-09-14T08:59:03.021 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:59:03.022 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:59:03.023 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:59:03.023 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sleep 1 2024-09-14T08:59:04.024 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T08:59:04.024 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ seq 0 3 2024-09-14T08:59:04.027 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:59:04.027 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_39 2024-09-14T08:59:04.028 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:0 2024-09-14T08:59:04.028 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:59:04.028 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:59:04.028 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T08:59:04.028 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:0 2024-09-14T08:59:04.028 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:59:04.028 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:59:04.028 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:59:04.028 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=0 2024-09-14T08:59:04.028 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T08:59:04.028 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:59:04.029 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:59:04.029 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=0 2024-09-14T08:59:04.029 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:59:04.029 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T08:59:04.029 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:0 2024-09-14T08:59:04.029 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:59:04.029 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:59:04.029 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T08:59:04.029 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:0 2024-09-14T08:59:04.029 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:59:04.029 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:59:04.029 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:59:04.030 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:59:04.030 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T08:59:04.030 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:59:04.030 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:59:04.030 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=0 2024-09-14T08:59:04.030 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=0 2024-09-14T08:59:04.030 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T08:59:04.049 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:59:04.049 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:59:04.049 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:59:04.049 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_39 2024-09-14T08:59:04.137 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_39'] 2024-09-14T08:59:04.137 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:59:04.137 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:59:04.140 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:59:04.140 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_39 2024-09-14T08:59:04.140 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:1 2024-09-14T08:59:04.140 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:59:04.140 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:59:04.140 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T08:59:04.140 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:1 2024-09-14T08:59:04.140 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:59:04.140 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:59:04.141 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:59:04.141 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=1 2024-09-14T08:59:04.141 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T08:59:04.141 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:59:04.141 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:59:04.141 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=1 2024-09-14T08:59:04.141 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:59:04.141 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T08:59:04.141 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:1 2024-09-14T08:59:04.141 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:59:04.141 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:59:04.141 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T08:59:04.141 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:1 2024-09-14T08:59:04.141 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:59:04.142 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:59:04.142 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:59:04.142 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:59:04.142 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T08:59:04.142 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:59:04.142 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:59:04.142 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=1 2024-09-14T08:59:04.142 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=1 2024-09-14T08:59:04.142 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T08:59:04.159 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:59:04.159 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:59:04.159 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:59:04.159 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_39 2024-09-14T08:59:04.237 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_39'] 2024-09-14T08:59:04.237 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:59:04.237 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:59:04.239 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:59:04.239 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_39 2024-09-14T08:59:04.240 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:2 2024-09-14T08:59:04.240 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:59:04.240 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:59:04.240 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-14T08:59:04.240 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:2 2024-09-14T08:59:04.240 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:59:04.240 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:59:04.240 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:59:04.240 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=2 2024-09-14T08:59:04.240 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T08:59:04.240 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:59:04.240 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:59:04.240 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=2 2024-09-14T08:59:04.241 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:59:04.241 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-14T08:59:04.241 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:2 2024-09-14T08:59:04.241 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:59:04.241 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:59:04.241 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T08:59:04.241 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:2 2024-09-14T08:59:04.241 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:59:04.241 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:59:04.241 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:59:04.241 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:59:04.241 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-14T08:59:04.241 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:59:04.241 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:59:04.241 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=2 2024-09-14T08:59:04.241 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=2 2024-09-14T08:59:04.242 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T08:59:04.258 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:59:04.258 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:59:04.258 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:59:04.259 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_39 2024-09-14T08:59:04.338 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_39'] 2024-09-14T08:59:04.338 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:59:04.338 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:59:04.341 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:59:04.342 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_39 2024-09-14T08:59:04.342 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:3 2024-09-14T08:59:04.342 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:59:04.342 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:59:04.342 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:3 cluster instance 2024-09-14T08:59:04.342 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:3 2024-09-14T08:59:04.342 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:59:04.342 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:59:04.342 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:59:04.342 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=3 2024-09-14T08:59:04.342 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-14T08:59:04.342 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:59:04.342 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:59:04.343 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=3 2024-09-14T08:59:04.343 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=3 2024-09-14T08:59:04.343 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:3 cluster1 2024-09-14T08:59:04.343 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:3 2024-09-14T08:59:04.343 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:59:04.343 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:59:04.343 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-14T08:59:04.343 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:3 2024-09-14T08:59:04.343 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:59:04.343 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:59:04.343 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:59:04.343 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=3 2024-09-14T08:59:04.343 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 3 = cluster1:3 ']' 2024-09-14T08:59:04.343 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:59:04.343 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:59:04.344 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=3 2024-09-14T08:59:04.344 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=3 2024-09-14T08:59:04.344 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-14T08:59:04.359 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T08:59:04.360 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T08:59:04.360 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T08:59:04.360 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_39 2024-09-14T08:59:04.440 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_39'] 2024-09-14T08:59:04.440 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:59:04.440 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:59:04.441 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:59:04.441 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sleep 2 2024-09-14T08:59:06.443 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T08:59:06.443 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ seq 0 3 2024-09-14T08:59:06.446 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:59:06.446 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_39 2024-09-14T08:59:06.446 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:0 2024-09-14T08:59:06.446 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:59:06.446 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:59:06.446 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T08:59:06.446 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:0 2024-09-14T08:59:06.446 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:59:06.446 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:59:06.446 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:59:06.446 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=0 2024-09-14T08:59:06.446 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T08:59:06.447 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:59:06.447 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:59:06.447 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=0 2024-09-14T08:59:06.447 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:59:06.447 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T08:59:06.447 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:0 2024-09-14T08:59:06.447 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:59:06.447 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:59:06.447 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T08:59:06.447 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:0 2024-09-14T08:59:06.447 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:59:06.447 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:59:06.447 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:59:06.447 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:59:06.448 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T08:59:06.448 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:59:06.448 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:59:06.448 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=0 2024-09-14T08:59:06.448 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=0 2024-09-14T08:59:06.448 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T08:59:06.467 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:59:06.467 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:59:06.467 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:59:06.467 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_39 2024-09-14T08:59:06.554 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_39'] 2024-09-14T08:59:06.554 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:59:06.554 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:59:06.557 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:59:06.557 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_39 2024-09-14T08:59:06.558 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:1 2024-09-14T08:59:06.558 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:59:06.558 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:59:06.558 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T08:59:06.558 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:1 2024-09-14T08:59:06.558 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:59:06.558 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:59:06.558 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:59:06.558 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=1 2024-09-14T08:59:06.558 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T08:59:06.558 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:59:06.558 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:59:06.558 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=1 2024-09-14T08:59:06.558 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:59:06.558 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T08:59:06.559 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:1 2024-09-14T08:59:06.559 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:59:06.559 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:59:06.559 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T08:59:06.559 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:1 2024-09-14T08:59:06.559 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:59:06.559 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:59:06.559 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:59:06.559 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:59:06.559 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T08:59:06.559 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:59:06.559 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:59:06.559 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=1 2024-09-14T08:59:06.559 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=1 2024-09-14T08:59:06.559 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T08:59:06.576 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:59:06.576 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:59:06.576 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:59:06.576 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_39 2024-09-14T08:59:06.654 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_39'] 2024-09-14T08:59:06.654 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:59:06.654 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:59:06.657 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:59:06.657 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_39 2024-09-14T08:59:06.657 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:2 2024-09-14T08:59:06.657 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:59:06.657 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:59:06.657 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-14T08:59:06.657 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:2 2024-09-14T08:59:06.657 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:59:06.657 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:59:06.657 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:59:06.657 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=2 2024-09-14T08:59:06.657 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T08:59:06.658 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:59:06.658 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:59:06.658 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=2 2024-09-14T08:59:06.658 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:59:06.658 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-14T08:59:06.658 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:2 2024-09-14T08:59:06.658 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:59:06.658 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:59:06.658 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T08:59:06.658 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:2 2024-09-14T08:59:06.658 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:59:06.658 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:59:06.658 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:59:06.658 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:59:06.659 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-14T08:59:06.659 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:59:06.659 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:59:06.659 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=2 2024-09-14T08:59:06.659 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=2 2024-09-14T08:59:06.659 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T08:59:06.675 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:59:06.675 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:59:06.675 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:59:06.675 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_39 2024-09-14T08:59:06.755 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_39'] 2024-09-14T08:59:06.755 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:59:06.755 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:59:06.758 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:59:06.758 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_39 2024-09-14T08:59:06.758 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:3 2024-09-14T08:59:06.758 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:59:06.758 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:59:06.758 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:3 cluster instance 2024-09-14T08:59:06.758 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:3 2024-09-14T08:59:06.759 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:59:06.759 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:59:06.759 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:59:06.759 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=3 2024-09-14T08:59:06.759 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-14T08:59:06.759 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:59:06.759 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:59:06.759 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=3 2024-09-14T08:59:06.759 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=3 2024-09-14T08:59:06.759 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:3 cluster1 2024-09-14T08:59:06.759 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:3 2024-09-14T08:59:06.759 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:59:06.759 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:59:06.759 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-14T08:59:06.759 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:3 2024-09-14T08:59:06.760 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:59:06.760 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:59:06.760 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:59:06.760 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=3 2024-09-14T08:59:06.760 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 3 = cluster1:3 ']' 2024-09-14T08:59:06.760 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:59:06.760 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:59:06.760 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=3 2024-09-14T08:59:06.760 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=3 2024-09-14T08:59:06.760 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-14T08:59:06.777 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T08:59:06.777 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T08:59:06.777 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T08:59:06.777 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_39 2024-09-14T08:59:06.856 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_39'] 2024-09-14T08:59:06.856 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:59:06.856 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:59:06.857 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:59:06.857 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sleep 4 2024-09-14T08:59:10.862 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T08:59:10.863 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ seq 0 3 2024-09-14T08:59:10.863 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:59:10.863 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_39 2024-09-14T08:59:10.863 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:0 2024-09-14T08:59:10.863 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:59:10.863 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:59:10.863 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T08:59:10.863 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:0 2024-09-14T08:59:10.863 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:59:10.863 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:59:10.863 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:59:10.863 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=0 2024-09-14T08:59:10.863 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T08:59:10.863 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:59:10.863 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:59:10.864 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=0 2024-09-14T08:59:10.864 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:59:10.864 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T08:59:10.864 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:0 2024-09-14T08:59:10.864 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:59:10.864 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:59:10.864 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T08:59:10.864 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:0 2024-09-14T08:59:10.864 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:59:10.864 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:59:10.864 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:59:10.864 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:59:10.864 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T08:59:10.864 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:59:10.864 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:59:10.864 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=0 2024-09-14T08:59:10.865 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=0 2024-09-14T08:59:10.865 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T08:59:10.885 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:59:10.885 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:59:10.885 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:59:10.885 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_39 2024-09-14T08:59:10.972 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_39'] 2024-09-14T08:59:10.972 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:59:10.972 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:59:10.975 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:59:10.975 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_39 2024-09-14T08:59:10.975 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:1 2024-09-14T08:59:10.975 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:59:10.975 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:59:10.975 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T08:59:10.975 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:1 2024-09-14T08:59:10.975 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:59:10.975 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:59:10.976 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:59:10.976 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=1 2024-09-14T08:59:10.976 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T08:59:10.976 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:59:10.976 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:59:10.976 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=1 2024-09-14T08:59:10.976 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:59:10.976 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T08:59:10.976 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:1 2024-09-14T08:59:10.977 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:59:10.977 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:59:10.977 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T08:59:10.977 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:1 2024-09-14T08:59:10.977 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:59:10.977 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:59:10.977 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:59:10.977 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:59:10.977 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T08:59:10.977 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:59:10.977 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:59:10.977 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=1 2024-09-14T08:59:10.977 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=1 2024-09-14T08:59:10.977 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T08:59:10.994 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:59:10.994 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:59:10.994 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:59:10.994 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_39 2024-09-14T08:59:11.070 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_39'] 2024-09-14T08:59:11.070 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:59:11.070 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:59:11.073 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:59:11.073 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_39 2024-09-14T08:59:11.073 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:2 2024-09-14T08:59:11.074 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:59:11.074 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:59:11.074 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-14T08:59:11.074 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:2 2024-09-14T08:59:11.074 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:59:11.074 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:59:11.074 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:59:11.074 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=2 2024-09-14T08:59:11.074 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T08:59:11.074 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:59:11.074 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:59:11.074 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=2 2024-09-14T08:59:11.074 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:59:11.074 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-14T08:59:11.074 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:2 2024-09-14T08:59:11.074 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:59:11.075 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:59:11.075 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T08:59:11.075 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:2 2024-09-14T08:59:11.075 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:59:11.075 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:59:11.075 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:59:11.075 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:59:11.075 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-14T08:59:11.075 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:59:11.075 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:59:11.075 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=2 2024-09-14T08:59:11.075 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=2 2024-09-14T08:59:11.075 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T08:59:11.092 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:59:11.092 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:59:11.092 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:59:11.092 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_39 2024-09-14T08:59:11.171 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_39'] 2024-09-14T08:59:11.172 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:59:11.172 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:59:11.175 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:59:11.175 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_39 2024-09-14T08:59:11.175 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:3 2024-09-14T08:59:11.175 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:59:11.175 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:59:11.175 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:3 cluster instance 2024-09-14T08:59:11.175 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:3 2024-09-14T08:59:11.175 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:59:11.175 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:59:11.176 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:59:11.176 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=3 2024-09-14T08:59:11.176 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-14T08:59:11.176 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:59:11.176 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:59:11.176 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=3 2024-09-14T08:59:11.176 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=3 2024-09-14T08:59:11.176 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:3 cluster1 2024-09-14T08:59:11.176 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:3 2024-09-14T08:59:11.176 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:59:11.176 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:59:11.176 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-14T08:59:11.176 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:3 2024-09-14T08:59:11.177 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:59:11.177 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:59:11.177 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:59:11.177 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=3 2024-09-14T08:59:11.177 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 3 = cluster1:3 ']' 2024-09-14T08:59:11.177 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:59:11.177 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:59:11.177 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=3 2024-09-14T08:59:11.177 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=3 2024-09-14T08:59:11.177 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-14T08:59:11.193 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T08:59:11.193 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T08:59:11.193 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T08:59:11.194 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_39 2024-09-14T08:59:11.273 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_39'] 2024-09-14T08:59:11.273 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:59:11.273 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:59:11.275 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:59:11.275 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sleep 8 2024-09-14T08:59:19.276 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T08:59:19.276 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ seq 0 3 2024-09-14T08:59:19.279 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:59:19.279 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_39 2024-09-14T08:59:19.279 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:0 2024-09-14T08:59:19.280 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:59:19.280 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:59:19.280 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T08:59:19.280 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:0 2024-09-14T08:59:19.280 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:59:19.280 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:59:19.280 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:59:19.280 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=0 2024-09-14T08:59:19.280 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T08:59:19.280 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:59:19.280 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:59:19.280 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=0 2024-09-14T08:59:19.280 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:59:19.280 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T08:59:19.281 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:0 2024-09-14T08:59:19.281 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:59:19.281 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:59:19.281 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T08:59:19.281 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:0 2024-09-14T08:59:19.281 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:59:19.281 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:59:19.281 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:59:19.281 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:59:19.281 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T08:59:19.281 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:59:19.281 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:59:19.281 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=0 2024-09-14T08:59:19.281 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=0 2024-09-14T08:59:19.281 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T08:59:19.300 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:59:19.300 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:59:19.300 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:59:19.300 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_39 2024-09-14T08:59:19.387 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_39'] 2024-09-14T08:59:19.387 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:59:19.387 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:59:19.390 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:59:19.391 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_39 2024-09-14T08:59:19.391 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:1 2024-09-14T08:59:19.391 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:59:19.391 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:59:19.391 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T08:59:19.391 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:1 2024-09-14T08:59:19.391 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:59:19.391 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:59:19.391 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:59:19.391 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=1 2024-09-14T08:59:19.391 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T08:59:19.391 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:59:19.392 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:59:19.392 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=1 2024-09-14T08:59:19.392 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:59:19.392 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T08:59:19.392 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:1 2024-09-14T08:59:19.392 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:59:19.392 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:59:19.392 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T08:59:19.392 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:1 2024-09-14T08:59:19.392 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:59:19.392 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:59:19.392 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:59:19.392 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:59:19.392 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T08:59:19.392 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:59:19.393 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:59:19.393 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=1 2024-09-14T08:59:19.393 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=1 2024-09-14T08:59:19.393 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T08:59:19.410 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:59:19.410 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:59:19.410 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:59:19.410 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_39 2024-09-14T08:59:19.499 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ return 0 2024-09-14T08:59:19.499 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T08:59:19.500 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ echo ' "state": "Replaying"' 2024-09-14T08:59:19.500 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep -i Replaying 2024-09-14T08:59:19.501 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: "state": "Replaying" 2024-09-14T08:59:19.502 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_state=started 2024-09-14T08:59:19.502 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test started = started 2024-09-14T08:59:19.502 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:59:19.503 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_39 2024-09-14T08:59:19.503 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:59:19.503 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:59:19.503 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:59:19.503 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_39 2024-09-14T08:59:19.503 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = journal ']' 2024-09-14T08:59:19.503 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = snapshot ']' 2024-09-14T08:59:19.503 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_39 2024-09-14T08:59:19.503 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:59:19.503 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:59:19.503 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:59:19.503 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_39 2024-09-14T08:59:19.503 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster2-mirror-image_39.status 2024-09-14T08:59:19.503 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster2-mirror-image_39.status 2024-09-14T08:59:19.504 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:59:19.505 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_39.status 2024-09-14T08:59:19.505 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster1-mirror-image_39.status 2024-09-14T08:59:19.505 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster1-mirror-image_39.status 2024-09-14T08:59:19.505 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:59:19.505 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_status_log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_39.status 2024-09-14T08:59:19.505 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ mirror_image_snapshot cluster2 mirror image_39 2024-09-14T08:59:19.505 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:59:19.505 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:59:19.505 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_39 2024-09-14T08:59:19.505 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_39 2024-09-14T08:59:19.625 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Snapshot ID: 162 2024-09-14T08:59:19.632 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_39 /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_39.status 2024-09-14T08:59:19.632 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:59:19.632 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:59:19.632 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_39 2024-09-14T08:59:19.632 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_39.status 2024-09-14T08:59:19.632 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_39 --format xml 2024-09-14T08:59:19.632 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:59:19.677 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-14T08:59:19.679 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snapshot_id=162 2024-09-14T08:59:19.680 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:59:19.680 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:59:19.680 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.2 2024-09-14T08:59:19.881 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_39 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_39.status 2024-09-14T08:59:19.881 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:59:19.881 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:59:19.881 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_39 2024-09-14T08:59:19.881 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_39.status 2024-09-14T08:59:19.881 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_39 --format xml 2024-09-14T08:59:19.881 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:59:19.930 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:59:19.931 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=73 2024-09-14T08:59:19.931 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 162 = 73 2024-09-14T08:59:19.931 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:59:19.932 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.4 2024-09-14T08:59:20.333 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_39 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_39.status 2024-09-14T08:59:20.333 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:59:20.333 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:59:20.333 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_39 2024-09-14T08:59:20.333 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_39.status 2024-09-14T08:59:20.333 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_39 --format xml 2024-09-14T08:59:20.333 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:59:20.382 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:59:20.384 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=73 2024-09-14T08:59:20.384 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 162 = 73 2024-09-14T08:59:20.384 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:59:20.384 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.8 2024-09-14T08:59:21.185 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_39 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_39.status 2024-09-14T08:59:21.185 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:59:21.185 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:59:21.185 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_39 2024-09-14T08:59:21.186 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_39.status 2024-09-14T08:59:21.186 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_39 --format xml 2024-09-14T08:59:21.186 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:59:21.240 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:59:21.240 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:59:21.241 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:59:21.241 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:59:21.241 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:59:21.242 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:59:21.242 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 162 = '' 2024-09-14T08:59:21.242 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:59:21.242 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1.6 2024-09-14T08:59:22.843 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_39 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_39.status 2024-09-14T08:59:22.843 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:59:22.844 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:59:22.844 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_39 2024-09-14T08:59:22.844 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_39.status 2024-09-14T08:59:22.844 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_39 --format xml 2024-09-14T08:59:22.844 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:59:22.895 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:59:22.897 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=162 2024-09-14T08:59:22.897 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 162 = 162 2024-09-14T08:59:22.897 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:59:22.897 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snap_present cluster1 mirror image_39 snap 2024-09-14T08:59:22.897 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:59:22.897 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:59:22.897 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_39 2024-09-14T08:59:22.898 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:59:22.898 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:59:22.898 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T08:59:22.898 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:59:23.898 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 info mirror/image_39@snap 2024-09-14T08:59:23.945 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_39': 2024-09-14T08:59:23.945 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T08:59:23.945 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:59:23.945 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 2 2024-09-14T08:59:23.945 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 129611f6a9f0 2024-09-14T08:59:23.945 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.129611f6a9f0 2024-09-14T08:59:23.945 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:59:23.946 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering, non-primary 2024-09-14T08:59:23.946 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:59:23.946 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:59:23.946 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:51:27 2024 2024-09-14T08:59:23.946 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:51:27 2024 2024-09-14T08:59:23.946 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:51:27 2024 2024-09-14T08:59:23.946 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: protected: False 2024-09-14T08:59:23.946 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:59:23.946 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:59:23.946 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: e2dde19c-e7be-4645-8807-0ea1479cbd41 2024-09-14T08:59:23.946 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: false 2024-09-14T08:59:23.949 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:59:23.949 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ compare_image_snaps mirror image_39 snap 2024-09-14T08:59:23.949 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:59:23.949 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_39 2024-09-14T08:59:23.949 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:59:23.949 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local ret=0 2024-09-14T08:59:23.949 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local rmt_export=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_39.export 2024-09-14T08:59:23.950 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local loc_export=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_39.export 2024-09-14T08:59:23.950 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_39.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_39.export 2024-09-14T08:59:23.950 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror export image_39@snap /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_39.export 2024-09-14T08:59:24.144 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T08:59:24.151 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 -p mirror export image_39@snap /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_39.export 2024-09-14T08:59:24.403 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T08:59:24.410 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cmp /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_39.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_39.export 2024-09-14T08:59:24.465 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_39.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_39.export 2024-09-14T08:59:24.496 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:59:24.496 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T08:59:24.496 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_40 2024-09-14T08:59:24.497 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_snap cluster2 mirror image_40 snap 2024-09-14T08:59:24.497 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:59:24.497 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:59:24.497 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_40 2024-09-14T08:59:24.497 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:59:24.497 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror snap create image_40@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T08:59:24.624 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_started cluster1 mirror image_40 2024-09-14T08:59:24.624 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:59:24.624 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:59:24.624 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_40 2024-09-14T08:59:24.624 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_state cluster1 mirror image_40 started 2024-09-14T08:59:24.624 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:59:24.624 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:59:24.624 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_40 2024-09-14T08:59:24.624 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=started 2024-09-14T08:59:24.624 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:59:24.625 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:59:24.625 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:59:25.625 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_replay_state cluster1 mirror image_40 started 2024-09-14T08:59:25.625 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:59:25.625 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:59:25.625 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_40 2024-09-14T08:59:25.625 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=started 2024-09-14T08:59:25.625 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_result 2024-09-14T08:59:25.625 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=stopped 2024-09-14T08:59:25.626 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_40 2024-09-14T08:59:25.626 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_instance=cluster1 2024-09-14T08:59:25.626 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ grep -i state 2024-09-14T08:59:25.626 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:59:25.626 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T08:59:25.627 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance=cluster1 2024-09-14T08:59:25.627 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local loop_instance 2024-09-14T08:59:25.627 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:59:25.627 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sleep 0 2024-09-14T08:59:25.627 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T08:59:25.627 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ seq 0 3 2024-09-14T08:59:25.629 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:59:25.630 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_40 2024-09-14T08:59:25.630 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:0 2024-09-14T08:59:25.630 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:59:25.630 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:59:25.630 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T08:59:25.630 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:0 2024-09-14T08:59:25.630 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:59:25.630 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:59:25.631 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:59:25.631 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=0 2024-09-14T08:59:25.631 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T08:59:25.631 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:59:25.631 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:59:25.631 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=0 2024-09-14T08:59:25.631 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:59:25.631 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T08:59:25.631 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:0 2024-09-14T08:59:25.631 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:59:25.631 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:59:25.631 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T08:59:25.631 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:0 2024-09-14T08:59:25.632 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:59:25.632 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:59:25.632 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:59:25.632 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:59:25.632 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T08:59:25.633 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:59:25.633 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:59:25.633 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=0 2024-09-14T08:59:25.633 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=0 2024-09-14T08:59:25.633 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T08:59:25.652 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:59:25.652 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:59:25.652 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:59:25.652 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_40 2024-09-14T08:59:25.748 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ return 0 2024-09-14T08:59:25.748 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T08:59:25.748 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ echo ' "state": "Replaying"' 2024-09-14T08:59:25.749 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep -i Replaying 2024-09-14T08:59:25.749 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: "state": "Replaying" 2024-09-14T08:59:25.751 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_state=started 2024-09-14T08:59:25.751 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test started = started 2024-09-14T08:59:25.751 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:59:25.751 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_40 2024-09-14T08:59:25.751 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:59:25.751 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:59:25.751 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:59:25.751 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_40 2024-09-14T08:59:25.751 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = journal ']' 2024-09-14T08:59:25.751 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = snapshot ']' 2024-09-14T08:59:25.752 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_40 2024-09-14T08:59:25.752 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:59:25.752 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:59:25.752 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:59:25.752 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_40 2024-09-14T08:59:25.752 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster2-mirror-image_40.status 2024-09-14T08:59:25.752 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster2-mirror-image_40.status 2024-09-14T08:59:25.752 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:59:25.753 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_40.status 2024-09-14T08:59:25.753 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster1-mirror-image_40.status 2024-09-14T08:59:25.753 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster1-mirror-image_40.status 2024-09-14T08:59:25.753 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:59:25.753 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_status_log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_40.status 2024-09-14T08:59:25.754 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ mirror_image_snapshot cluster2 mirror image_40 2024-09-14T08:59:25.754 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:59:25.754 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:59:25.754 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_40 2024-09-14T08:59:25.754 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_40 2024-09-14T08:59:26.637 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Snapshot ID: 164 2024-09-14T08:59:26.643 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_40 /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_40.status 2024-09-14T08:59:26.644 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:59:26.644 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:59:26.644 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_40 2024-09-14T08:59:26.644 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_40.status 2024-09-14T08:59:26.644 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_40 --format xml 2024-09-14T08:59:26.644 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:59:26.692 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-14T08:59:26.695 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snapshot_id=164 2024-09-14T08:59:26.695 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:59:26.695 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:59:26.695 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.2 2024-09-14T08:59:26.896 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_40 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_40.status 2024-09-14T08:59:26.896 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:59:26.896 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:59:26.896 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_40 2024-09-14T08:59:26.896 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_40.status 2024-09-14T08:59:26.896 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_40 --format xml 2024-09-14T08:59:26.896 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:59:26.946 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:59:26.948 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=74 2024-09-14T08:59:26.948 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 164 = 74 2024-09-14T08:59:26.948 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:59:26.948 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.4 2024-09-14T08:59:27.350 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_40 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_40.status 2024-09-14T08:59:27.350 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:59:27.350 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:59:27.350 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_40 2024-09-14T08:59:27.350 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_40.status 2024-09-14T08:59:27.350 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_40 --format xml 2024-09-14T08:59:27.350 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:59:27.401 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:59:27.403 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=74 2024-09-14T08:59:27.403 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 164 = 74 2024-09-14T08:59:27.403 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:59:27.403 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.8 2024-09-14T08:59:28.204 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_40 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_40.status 2024-09-14T08:59:28.204 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:59:28.205 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:59:28.205 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_40 2024-09-14T08:59:28.205 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_40.status 2024-09-14T08:59:28.205 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_40 --format xml 2024-09-14T08:59:28.205 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:59:28.259 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:59:28.259 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:59:28.261 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:59:28.261 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:59:28.261 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:59:28.261 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:59:28.262 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 164 = '' 2024-09-14T08:59:28.262 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:59:28.262 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1.6 2024-09-14T08:59:29.863 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_40 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_40.status 2024-09-14T08:59:29.863 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:59:29.863 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:59:29.863 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_40 2024-09-14T08:59:29.863 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_40.status 2024-09-14T08:59:29.863 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_40 --format xml 2024-09-14T08:59:29.863 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:59:29.917 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:59:29.919 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=164 2024-09-14T08:59:29.919 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 164 = 164 2024-09-14T08:59:29.919 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:59:29.919 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snap_present cluster1 mirror image_40 snap 2024-09-14T08:59:29.919 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:59:29.919 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:59:29.919 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_40 2024-09-14T08:59:29.920 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:59:29.920 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:59:29.920 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T08:59:29.920 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:59:30.920 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 info mirror/image_40@snap 2024-09-14T08:59:30.967 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_40': 2024-09-14T08:59:30.968 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T08:59:30.968 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:59:30.968 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 2 2024-09-14T08:59:30.968 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 12a0a2f45c76 2024-09-14T08:59:30.968 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.12a0a2f45c76 2024-09-14T08:59:30.968 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:59:30.968 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering, non-primary 2024-09-14T08:59:30.968 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:59:30.968 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:59:30.968 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:51:40 2024 2024-09-14T08:59:30.968 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:51:40 2024 2024-09-14T08:59:30.968 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:51:40 2024 2024-09-14T08:59:30.968 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: protected: False 2024-09-14T08:59:30.968 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:59:30.969 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:59:30.969 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: 3327bc1e-5162-4a18-8a41-c4a1c978f5cf 2024-09-14T08:59:30.969 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: false 2024-09-14T08:59:30.972 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:59:30.972 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ compare_image_snaps mirror image_40 snap 2024-09-14T08:59:30.972 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:59:30.972 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_40 2024-09-14T08:59:30.972 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:59:30.972 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local ret=0 2024-09-14T08:59:30.972 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local rmt_export=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_40.export 2024-09-14T08:59:30.972 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local loc_export=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_40.export 2024-09-14T08:59:30.972 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_40.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_40.export 2024-09-14T08:59:30.973 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror export image_40@snap /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_40.export 2024-09-14T08:59:31.188 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T08:59:31.196 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 -p mirror export image_40@snap /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_40.export 2024-09-14T08:59:31.463 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T08:59:31.470 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cmp /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_40.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_40.export 2024-09-14T08:59:31.524 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_40.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_40.export 2024-09-14T08:59:31.555 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:59:31.556 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T08:59:31.556 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_41 2024-09-14T08:59:31.556 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_snap cluster2 mirror image_41 snap 2024-09-14T08:59:31.556 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:59:31.556 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:59:31.556 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_41 2024-09-14T08:59:31.556 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:59:31.556 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror snap create image_41@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T08:59:31.640 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_started cluster1 mirror image_41 2024-09-14T08:59:31.640 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:59:31.640 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:59:31.640 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_41 2024-09-14T08:59:31.640 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_state cluster1 mirror image_41 started 2024-09-14T08:59:31.640 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:59:31.640 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:59:31.640 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_41 2024-09-14T08:59:31.640 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=started 2024-09-14T08:59:31.640 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:59:31.640 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:59:31.640 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:59:32.642 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_replay_state cluster1 mirror image_41 started 2024-09-14T08:59:32.642 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:59:32.642 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:59:32.642 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_41 2024-09-14T08:59:32.642 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=started 2024-09-14T08:59:32.642 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_result 2024-09-14T08:59:32.643 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=stopped 2024-09-14T08:59:32.643 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_41 2024-09-14T08:59:32.643 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_instance=cluster1 2024-09-14T08:59:32.643 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:59:32.643 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ grep -i state 2024-09-14T08:59:32.643 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T08:59:32.643 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance=cluster1 2024-09-14T08:59:32.643 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local loop_instance 2024-09-14T08:59:32.643 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:59:32.643 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sleep 0 2024-09-14T08:59:32.646 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T08:59:32.646 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ seq 0 3 2024-09-14T08:59:32.646 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:59:32.647 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_41 2024-09-14T08:59:32.647 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:0 2024-09-14T08:59:32.647 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:59:32.647 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:59:32.647 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T08:59:32.647 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:0 2024-09-14T08:59:32.647 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:59:32.647 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:59:32.647 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:59:32.647 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=0 2024-09-14T08:59:32.647 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T08:59:32.647 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:59:32.647 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:59:32.648 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=0 2024-09-14T08:59:32.648 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:59:32.648 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T08:59:32.648 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:0 2024-09-14T08:59:32.648 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:59:32.648 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:59:32.648 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T08:59:32.648 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:0 2024-09-14T08:59:32.648 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:59:32.648 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:59:32.648 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:59:32.649 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:59:32.649 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T08:59:32.649 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:59:32.649 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:59:32.649 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=0 2024-09-14T08:59:32.649 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=0 2024-09-14T08:59:32.649 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T08:59:32.666 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:59:32.666 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:59:32.666 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:59:32.667 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_41 2024-09-14T08:59:32.750 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_41'] 2024-09-14T08:59:32.750 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:59:32.750 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:59:32.753 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:59:32.754 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_41 2024-09-14T08:59:32.754 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:1 2024-09-14T08:59:32.754 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:59:32.754 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:59:32.754 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T08:59:32.754 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:1 2024-09-14T08:59:32.754 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:59:32.754 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:59:32.754 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:59:32.754 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=1 2024-09-14T08:59:32.755 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T08:59:32.755 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:59:32.755 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:59:32.755 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=1 2024-09-14T08:59:32.755 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:59:32.755 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T08:59:32.755 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:1 2024-09-14T08:59:32.755 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:59:32.755 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:59:32.755 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T08:59:32.755 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:1 2024-09-14T08:59:32.755 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:59:32.755 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:59:32.755 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:59:32.756 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:59:32.756 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T08:59:32.756 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:59:32.756 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:59:32.756 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=1 2024-09-14T08:59:32.756 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=1 2024-09-14T08:59:32.756 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T08:59:32.772 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:59:32.772 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:59:32.772 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:59:32.772 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_41 2024-09-14T08:59:32.860 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ return 0 2024-09-14T08:59:32.860 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T08:59:32.860 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ echo ' "state": "Replaying"' 2024-09-14T08:59:32.861 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep -i Replaying 2024-09-14T08:59:32.861 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: "state": "Replaying" 2024-09-14T08:59:32.863 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_state=started 2024-09-14T08:59:32.863 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test started = started 2024-09-14T08:59:32.863 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:59:32.863 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_41 2024-09-14T08:59:32.863 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:59:32.863 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:59:32.863 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:59:32.863 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_41 2024-09-14T08:59:32.863 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = journal ']' 2024-09-14T08:59:32.863 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = snapshot ']' 2024-09-14T08:59:32.863 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_41 2024-09-14T08:59:32.863 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:59:32.864 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:59:32.864 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:59:32.864 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_41 2024-09-14T08:59:32.864 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster2-mirror-image_41.status 2024-09-14T08:59:32.864 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster2-mirror-image_41.status 2024-09-14T08:59:32.864 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:59:32.864 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_41.status 2024-09-14T08:59:32.865 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster1-mirror-image_41.status 2024-09-14T08:59:32.865 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster1-mirror-image_41.status 2024-09-14T08:59:32.865 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:59:32.865 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_status_log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_41.status 2024-09-14T08:59:32.866 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ mirror_image_snapshot cluster2 mirror image_41 2024-09-14T08:59:32.866 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:59:32.866 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:59:32.866 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_41 2024-09-14T08:59:32.866 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_41 2024-09-14T08:59:33.648 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Snapshot ID: 166 2024-09-14T08:59:33.656 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_41 /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_41.status 2024-09-14T08:59:33.656 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:59:33.656 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:59:33.656 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_41 2024-09-14T08:59:33.656 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_41.status 2024-09-14T08:59:33.656 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_41 --format xml 2024-09-14T08:59:33.656 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:59:33.705 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-14T08:59:33.707 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snapshot_id=166 2024-09-14T08:59:33.707 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:59:33.707 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:59:33.707 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.2 2024-09-14T08:59:33.909 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_41 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_41.status 2024-09-14T08:59:33.909 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:59:33.909 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:59:33.909 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_41 2024-09-14T08:59:33.909 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_41.status 2024-09-14T08:59:33.909 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_41 --format xml 2024-09-14T08:59:33.909 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:59:33.960 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:59:33.962 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=75 2024-09-14T08:59:33.962 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 166 = 75 2024-09-14T08:59:33.962 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:59:33.962 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.4 2024-09-14T08:59:34.363 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_41 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_41.status 2024-09-14T08:59:34.363 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:59:34.363 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:59:34.364 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_41 2024-09-14T08:59:34.364 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_41.status 2024-09-14T08:59:34.364 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_41 --format xml 2024-09-14T08:59:34.364 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:59:34.413 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:59:34.415 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=75 2024-09-14T08:59:34.415 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 166 = 75 2024-09-14T08:59:34.415 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:59:34.415 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.8 2024-09-14T08:59:35.216 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_41 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_41.status 2024-09-14T08:59:35.217 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:59:35.217 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:59:35.217 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_41 2024-09-14T08:59:35.217 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_41.status 2024-09-14T08:59:35.217 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_41 --format xml 2024-09-14T08:59:35.217 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:59:35.273 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:59:35.273 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:59:35.275 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:59:35.275 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:59:35.275 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:59:35.275 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:59:35.275 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 166 = '' 2024-09-14T08:59:35.275 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:59:35.275 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1.6 2024-09-14T08:59:36.877 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_41 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_41.status 2024-09-14T08:59:36.877 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:59:36.877 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:59:36.877 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_41 2024-09-14T08:59:36.877 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_41.status 2024-09-14T08:59:36.877 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_41 --format xml 2024-09-14T08:59:36.877 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:59:36.925 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:59:36.928 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=166 2024-09-14T08:59:36.928 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 166 = 166 2024-09-14T08:59:36.928 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:59:36.928 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snap_present cluster1 mirror image_41 snap 2024-09-14T08:59:36.928 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:59:36.928 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:59:36.928 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_41 2024-09-14T08:59:36.928 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:59:36.928 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:59:36.928 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T08:59:36.928 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:59:37.929 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 info mirror/image_41@snap 2024-09-14T08:59:37.975 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_41': 2024-09-14T08:59:37.975 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T08:59:37.975 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:59:37.975 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 2 2024-09-14T08:59:37.975 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 10bdec10bf1 2024-09-14T08:59:37.975 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.10bdec10bf1 2024-09-14T08:59:37.976 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:59:37.976 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering, non-primary 2024-09-14T08:59:37.976 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:59:37.976 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:59:37.976 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:51:54 2024 2024-09-14T08:59:37.976 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:51:54 2024 2024-09-14T08:59:37.976 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:51:54 2024 2024-09-14T08:59:37.976 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: protected: False 2024-09-14T08:59:37.976 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:59:37.976 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:59:37.976 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: b734de70-db13-448a-a651-6020a282fb17 2024-09-14T08:59:37.976 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: false 2024-09-14T08:59:37.979 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:59:37.979 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ compare_image_snaps mirror image_41 snap 2024-09-14T08:59:37.979 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:59:37.979 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_41 2024-09-14T08:59:37.979 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:59:37.979 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local ret=0 2024-09-14T08:59:37.979 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local rmt_export=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_41.export 2024-09-14T08:59:37.979 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local loc_export=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_41.export 2024-09-14T08:59:37.979 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_41.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_41.export 2024-09-14T08:59:37.980 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror export image_41@snap /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_41.export 2024-09-14T08:59:38.180 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T08:59:38.185 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 -p mirror export image_41@snap /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_41.export 2024-09-14T08:59:38.450 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T08:59:38.458 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cmp /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_41.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_41.export 2024-09-14T08:59:38.513 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_41.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_41.export 2024-09-14T08:59:38.546 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:59:38.546 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T08:59:38.546 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_42 2024-09-14T08:59:38.546 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_snap cluster2 mirror image_42 snap 2024-09-14T08:59:38.546 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:59:38.546 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:59:38.546 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_42 2024-09-14T08:59:38.547 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:59:38.547 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror snap create image_42@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T08:59:38.642 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_started cluster1 mirror image_42 2024-09-14T08:59:38.642 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:59:38.642 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:59:38.642 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_42 2024-09-14T08:59:38.643 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_state cluster1 mirror image_42 started 2024-09-14T08:59:38.643 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:59:38.643 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:59:38.643 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_42 2024-09-14T08:59:38.643 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=started 2024-09-14T08:59:38.643 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:59:38.643 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:59:38.643 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:59:39.643 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_replay_state cluster1 mirror image_42 started 2024-09-14T08:59:39.643 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:59:39.644 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:59:39.644 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_42 2024-09-14T08:59:39.644 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=started 2024-09-14T08:59:39.644 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_result 2024-09-14T08:59:39.644 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=stopped 2024-09-14T08:59:39.647 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_42 2024-09-14T08:59:39.647 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_instance=cluster1 2024-09-14T08:59:39.647 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:59:39.647 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ grep -i state 2024-09-14T08:59:39.648 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T08:59:39.648 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance=cluster1 2024-09-14T08:59:39.648 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local loop_instance 2024-09-14T08:59:39.648 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:59:39.648 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sleep 0 2024-09-14T08:59:39.648 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T08:59:39.648 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ seq 0 3 2024-09-14T08:59:39.648 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:59:39.648 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_42 2024-09-14T08:59:39.648 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:0 2024-09-14T08:59:39.648 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:59:39.648 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:59:39.648 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T08:59:39.649 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:0 2024-09-14T08:59:39.649 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:59:39.649 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:59:39.649 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:59:39.649 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=0 2024-09-14T08:59:39.649 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T08:59:39.649 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:59:39.649 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:59:39.649 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=0 2024-09-14T08:59:39.649 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:59:39.649 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T08:59:39.649 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:0 2024-09-14T08:59:39.649 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:59:39.649 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:59:39.650 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T08:59:39.650 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:0 2024-09-14T08:59:39.650 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:59:39.650 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:59:39.650 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:59:39.650 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:59:39.650 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T08:59:39.650 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:59:39.650 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:59:39.650 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=0 2024-09-14T08:59:39.650 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=0 2024-09-14T08:59:39.650 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T08:59:39.667 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:59:39.667 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:59:39.667 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:59:39.667 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_42 2024-09-14T08:59:39.749 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_42'] 2024-09-14T08:59:39.749 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:59:39.749 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:59:39.751 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:59:39.752 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_42 2024-09-14T08:59:39.752 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:1 2024-09-14T08:59:39.752 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:59:39.752 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:59:39.752 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T08:59:39.752 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:1 2024-09-14T08:59:39.752 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:59:39.752 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:59:39.752 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:59:39.752 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=1 2024-09-14T08:59:39.752 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T08:59:39.752 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:59:39.752 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:59:39.753 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=1 2024-09-14T08:59:39.753 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:59:39.753 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T08:59:39.753 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:1 2024-09-14T08:59:39.753 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:59:39.753 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:59:39.753 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T08:59:39.753 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:1 2024-09-14T08:59:39.753 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:59:39.753 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:59:39.753 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:59:39.753 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:59:39.753 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T08:59:39.753 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:59:39.753 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:59:39.754 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=1 2024-09-14T08:59:39.754 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=1 2024-09-14T08:59:39.754 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T08:59:39.770 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:59:39.770 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:59:39.771 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:59:39.771 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_42 2024-09-14T08:59:39.858 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ return 0 2024-09-14T08:59:39.858 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T08:59:39.859 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ echo ' "state": "Replaying"' 2024-09-14T08:59:39.859 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep -i Replaying 2024-09-14T08:59:39.859 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: "state": "Replaying" 2024-09-14T08:59:39.860 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_state=started 2024-09-14T08:59:39.861 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test started = started 2024-09-14T08:59:39.861 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:59:39.861 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_42 2024-09-14T08:59:39.861 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:59:39.861 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:59:39.861 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:59:39.861 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_42 2024-09-14T08:59:39.861 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = journal ']' 2024-09-14T08:59:39.861 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = snapshot ']' 2024-09-14T08:59:39.861 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_42 2024-09-14T08:59:39.861 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:59:39.861 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:59:39.861 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:59:39.861 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_42 2024-09-14T08:59:39.862 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster2-mirror-image_42.status 2024-09-14T08:59:39.862 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster2-mirror-image_42.status 2024-09-14T08:59:39.862 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:59:39.862 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_42.status 2024-09-14T08:59:39.862 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster1-mirror-image_42.status 2024-09-14T08:59:39.863 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster1-mirror-image_42.status 2024-09-14T08:59:39.863 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:59:39.863 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_status_log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_42.status 2024-09-14T08:59:39.863 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ mirror_image_snapshot cluster2 mirror image_42 2024-09-14T08:59:39.864 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:59:39.864 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:59:39.864 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_42 2024-09-14T08:59:39.864 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_42 2024-09-14T08:59:40.654 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Snapshot ID: 168 2024-09-14T08:59:40.660 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_42 /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_42.status 2024-09-14T08:59:40.661 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:59:40.661 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:59:40.661 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_42 2024-09-14T08:59:40.661 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_42.status 2024-09-14T08:59:40.661 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_42 --format xml 2024-09-14T08:59:40.661 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:59:40.709 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-14T08:59:40.711 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snapshot_id=168 2024-09-14T08:59:40.711 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:59:40.711 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:59:40.711 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.2 2024-09-14T08:59:40.913 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_42 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_42.status 2024-09-14T08:59:40.913 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:59:40.913 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:59:40.913 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_42 2024-09-14T08:59:40.913 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_42.status 2024-09-14T08:59:40.913 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_42 --format xml 2024-09-14T08:59:40.913 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:59:40.962 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:59:40.964 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=76 2024-09-14T08:59:40.964 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 168 = 76 2024-09-14T08:59:40.965 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:59:40.965 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.4 2024-09-14T08:59:41.366 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_42 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_42.status 2024-09-14T08:59:41.366 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:59:41.366 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:59:41.366 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_42 2024-09-14T08:59:41.366 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_42.status 2024-09-14T08:59:41.366 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_42 --format xml 2024-09-14T08:59:41.366 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:59:41.417 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:59:41.419 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=76 2024-09-14T08:59:41.419 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 168 = 76 2024-09-14T08:59:41.419 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:59:41.419 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.8 2024-09-14T08:59:42.221 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_42 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_42.status 2024-09-14T08:59:42.221 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:59:42.221 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:59:42.221 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_42 2024-09-14T08:59:42.221 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_42.status 2024-09-14T08:59:42.221 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_42 --format xml 2024-09-14T08:59:42.221 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:59:42.275 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:59:42.276 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:59:42.278 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:59:42.278 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:59:42.278 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:59:42.278 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:59:42.278 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 168 = '' 2024-09-14T08:59:42.278 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:59:42.278 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1.6 2024-09-14T08:59:43.879 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_42 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_42.status 2024-09-14T08:59:43.879 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:59:43.879 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:59:43.879 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_42 2024-09-14T08:59:43.880 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_42.status 2024-09-14T08:59:43.880 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_42 --format xml 2024-09-14T08:59:43.880 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:59:43.937 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:59:43.937 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=168 2024-09-14T08:59:43.937 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 168 = 168 2024-09-14T08:59:43.937 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:59:43.938 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snap_present cluster1 mirror image_42 snap 2024-09-14T08:59:43.938 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:59:43.938 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:59:43.938 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_42 2024-09-14T08:59:43.938 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:59:43.938 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:59:43.938 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T08:59:43.938 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:59:44.938 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 info mirror/image_42@snap 2024-09-14T08:59:44.984 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_42': 2024-09-14T08:59:44.984 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T08:59:44.984 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:59:44.984 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 2 2024-09-14T08:59:44.984 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 11897d1cec1a 2024-09-14T08:59:44.984 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.11897d1cec1a 2024-09-14T08:59:44.984 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:59:44.984 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering, non-primary 2024-09-14T08:59:44.985 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:59:44.985 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:59:44.985 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:52:08 2024 2024-09-14T08:59:44.985 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:52:08 2024 2024-09-14T08:59:44.985 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:52:08 2024 2024-09-14T08:59:44.985 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: protected: False 2024-09-14T08:59:44.985 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:59:44.985 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:59:44.985 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: 071d6746-d644-418c-9496-15e5e66ef4f1 2024-09-14T08:59:44.985 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: false 2024-09-14T08:59:44.988 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:59:44.988 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ compare_image_snaps mirror image_42 snap 2024-09-14T08:59:44.988 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:59:44.988 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_42 2024-09-14T08:59:44.988 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:59:44.988 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local ret=0 2024-09-14T08:59:44.988 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local rmt_export=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_42.export 2024-09-14T08:59:44.989 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local loc_export=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_42.export 2024-09-14T08:59:44.989 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_42.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_42.export 2024-09-14T08:59:44.989 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror export image_42@snap /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_42.export 2024-09-14T08:59:45.188 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T08:59:45.196 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 -p mirror export image_42@snap /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_42.export 2024-09-14T08:59:45.439 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T08:59:45.445 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cmp /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_42.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_42.export 2024-09-14T08:59:45.499 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_42.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_42.export 2024-09-14T08:59:45.532 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:59:45.532 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T08:59:45.532 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_43 2024-09-14T08:59:45.532 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_snap cluster2 mirror image_43 snap 2024-09-14T08:59:45.532 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:59:45.532 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:59:45.532 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_43 2024-09-14T08:59:45.532 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:59:45.532 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror snap create image_43@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T08:59:45.666 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_started cluster1 mirror image_43 2024-09-14T08:59:45.666 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:59:45.666 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:59:45.666 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_43 2024-09-14T08:59:45.666 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_state cluster1 mirror image_43 started 2024-09-14T08:59:45.666 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:59:45.666 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:59:45.666 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_43 2024-09-14T08:59:45.666 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=started 2024-09-14T08:59:45.666 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:59:45.666 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:59:45.667 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:59:46.667 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_replay_state cluster1 mirror image_43 started 2024-09-14T08:59:46.667 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:59:46.668 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:59:46.668 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_43 2024-09-14T08:59:46.668 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=started 2024-09-14T08:59:46.668 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_result 2024-09-14T08:59:46.668 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=stopped 2024-09-14T08:59:46.668 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_43 2024-09-14T08:59:46.668 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_instance=cluster1 2024-09-14T08:59:46.668 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:59:46.668 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ grep -i state 2024-09-14T08:59:46.668 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T08:59:46.668 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance=cluster1 2024-09-14T08:59:46.668 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local loop_instance 2024-09-14T08:59:46.668 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:59:46.668 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sleep 0 2024-09-14T08:59:46.670 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T08:59:46.670 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ seq 0 3 2024-09-14T08:59:46.671 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:59:46.671 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_43 2024-09-14T08:59:46.671 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:0 2024-09-14T08:59:46.671 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:59:46.671 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:59:46.671 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T08:59:46.671 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:0 2024-09-14T08:59:46.671 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:59:46.671 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:59:46.671 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:59:46.671 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=0 2024-09-14T08:59:46.671 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T08:59:46.671 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:59:46.671 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:59:46.672 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=0 2024-09-14T08:59:46.672 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:59:46.672 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T08:59:46.672 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:0 2024-09-14T08:59:46.672 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:59:46.672 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:59:46.672 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T08:59:46.672 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:0 2024-09-14T08:59:46.672 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:59:46.672 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:59:46.672 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:59:46.672 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:59:46.673 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T08:59:46.673 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:59:46.673 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:59:46.673 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=0 2024-09-14T08:59:46.673 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=0 2024-09-14T08:59:46.673 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T08:59:46.691 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:59:46.691 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:59:46.691 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:59:46.691 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_43 2024-09-14T08:59:46.774 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_43'] 2024-09-14T08:59:46.774 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:59:46.774 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:59:46.777 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:59:46.777 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_43 2024-09-14T08:59:46.777 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:1 2024-09-14T08:59:46.778 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:59:46.778 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:59:46.778 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T08:59:46.778 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:1 2024-09-14T08:59:46.778 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:59:46.778 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:59:46.778 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:59:46.778 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=1 2024-09-14T08:59:46.778 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T08:59:46.778 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:59:46.778 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:59:46.778 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=1 2024-09-14T08:59:46.778 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:59:46.778 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T08:59:46.778 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:1 2024-09-14T08:59:46.778 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:59:46.779 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:59:46.779 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T08:59:46.779 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:1 2024-09-14T08:59:46.779 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:59:46.779 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:59:46.779 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:59:46.779 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:59:46.779 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T08:59:46.779 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:59:46.779 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:59:46.779 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=1 2024-09-14T08:59:46.779 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=1 2024-09-14T08:59:46.779 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T08:59:46.797 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:59:46.797 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:59:46.798 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:59:46.798 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_43 2024-09-14T08:59:46.885 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ return 0 2024-09-14T08:59:46.886 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T08:59:46.886 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ echo ' "state": "Replaying"' 2024-09-14T08:59:46.886 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep -i Replaying 2024-09-14T08:59:46.886 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: "state": "Replaying" 2024-09-14T08:59:46.888 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_state=started 2024-09-14T08:59:46.888 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test started = started 2024-09-14T08:59:46.888 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:59:46.888 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_43 2024-09-14T08:59:46.888 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:59:46.888 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:59:46.888 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:59:46.888 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_43 2024-09-14T08:59:46.888 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = journal ']' 2024-09-14T08:59:46.888 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = snapshot ']' 2024-09-14T08:59:46.888 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_43 2024-09-14T08:59:46.889 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:59:46.889 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:59:46.889 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:59:46.889 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_43 2024-09-14T08:59:46.889 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster2-mirror-image_43.status 2024-09-14T08:59:46.889 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster2-mirror-image_43.status 2024-09-14T08:59:46.889 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:59:46.890 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_43.status 2024-09-14T08:59:46.890 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster1-mirror-image_43.status 2024-09-14T08:59:46.890 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster1-mirror-image_43.status 2024-09-14T08:59:46.890 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:59:46.890 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_status_log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_43.status 2024-09-14T08:59:46.891 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ mirror_image_snapshot cluster2 mirror image_43 2024-09-14T08:59:46.891 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:59:46.891 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:59:46.891 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_43 2024-09-14T08:59:46.891 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_43 2024-09-14T08:59:47.678 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Snapshot ID: 170 2024-09-14T08:59:47.684 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_43 /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_43.status 2024-09-14T08:59:47.684 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:59:47.684 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:59:47.684 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_43 2024-09-14T08:59:47.684 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_43.status 2024-09-14T08:59:47.685 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_43 --format xml 2024-09-14T08:59:47.685 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:59:47.932 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-14T08:59:47.934 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snapshot_id=170 2024-09-14T08:59:47.934 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:59:47.934 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:59:47.934 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.2 2024-09-14T08:59:48.135 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_43 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_43.status 2024-09-14T08:59:48.135 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:59:48.135 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:59:48.135 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_43 2024-09-14T08:59:48.135 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_43.status 2024-09-14T08:59:48.135 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_43 --format xml 2024-09-14T08:59:48.135 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:59:48.184 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:59:48.186 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=77 2024-09-14T08:59:48.186 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 170 = 77 2024-09-14T08:59:48.186 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:59:48.186 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.4 2024-09-14T08:59:48.587 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_43 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_43.status 2024-09-14T08:59:48.587 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:59:48.587 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:59:48.587 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_43 2024-09-14T08:59:48.588 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_43.status 2024-09-14T08:59:48.588 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_43 --format xml 2024-09-14T08:59:48.588 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:59:48.639 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:59:48.640 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:59:48.641 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:59:48.641 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:59:48.641 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:59:48.641 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:59:48.641 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 170 = '' 2024-09-14T08:59:48.641 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:59:48.641 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.8 2024-09-14T08:59:49.443 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_43 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_43.status 2024-09-14T08:59:49.443 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:59:49.443 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:59:49.443 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_43 2024-09-14T08:59:49.443 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_43.status 2024-09-14T08:59:49.443 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_43 --format xml 2024-09-14T08:59:49.443 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:59:49.574 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:59:49.574 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:59:49.575 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:59:49.575 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:59:49.576 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:59:49.576 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:59:49.576 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 170 = '' 2024-09-14T08:59:49.576 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:59:49.576 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1.6 2024-09-14T08:59:51.177 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_43 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_43.status 2024-09-14T08:59:51.178 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:59:51.178 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:59:51.178 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_43 2024-09-14T08:59:51.178 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_43.status 2024-09-14T08:59:51.178 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_43 --format xml 2024-09-14T08:59:51.178 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:59:51.233 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:59:51.236 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=170 2024-09-14T08:59:51.236 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 170 = 170 2024-09-14T08:59:51.236 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:59:51.236 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snap_present cluster1 mirror image_43 snap 2024-09-14T08:59:51.236 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:59:51.236 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:59:51.236 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_43 2024-09-14T08:59:51.236 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:59:51.236 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:59:51.236 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T08:59:51.237 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:59:52.237 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 info mirror/image_43@snap 2024-09-14T08:59:52.284 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_43': 2024-09-14T08:59:52.284 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T08:59:52.284 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T08:59:52.285 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 2 2024-09-14T08:59:52.285 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 1296e246995d 2024-09-14T08:59:52.285 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.1296e246995d 2024-09-14T08:59:52.285 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T08:59:52.285 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering, non-primary 2024-09-14T08:59:52.285 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T08:59:52.285 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T08:59:52.285 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:52:18 2024 2024-09-14T08:59:52.285 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:52:18 2024 2024-09-14T08:59:52.285 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:52:18 2024 2024-09-14T08:59:52.285 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: protected: False 2024-09-14T08:59:52.285 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T08:59:52.285 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T08:59:52.285 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: 3589cded-f286-4a62-a351-1f0a15e1b107 2024-09-14T08:59:52.285 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: false 2024-09-14T08:59:52.288 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:59:52.288 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ compare_image_snaps mirror image_43 snap 2024-09-14T08:59:52.288 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:59:52.288 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_43 2024-09-14T08:59:52.288 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:59:52.289 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local ret=0 2024-09-14T08:59:52.289 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local rmt_export=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_43.export 2024-09-14T08:59:52.289 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local loc_export=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_43.export 2024-09-14T08:59:52.289 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_43.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_43.export 2024-09-14T08:59:52.289 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror export image_43@snap /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_43.export 2024-09-14T08:59:52.494 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T08:59:52.503 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 -p mirror export image_43@snap /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_43.export 2024-09-14T08:59:52.765 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T08:59:52.772 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cmp /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_43.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_43.export 2024-09-14T08:59:52.826 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_43.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_43.export 2024-09-14T08:59:52.858 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:59:52.858 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T08:59:52.858 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_44 2024-09-14T08:59:52.858 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_snap cluster2 mirror image_44 snap 2024-09-14T08:59:52.858 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:59:52.858 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:59:52.858 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_44 2024-09-14T08:59:52.858 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:59:52.858 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror snap create image_44@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T08:59:53.685 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_started cluster1 mirror image_44 2024-09-14T08:59:53.685 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:59:53.685 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:59:53.685 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_44 2024-09-14T08:59:53.685 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_state cluster1 mirror image_44 started 2024-09-14T08:59:53.685 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:59:53.685 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:59:53.685 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_44 2024-09-14T08:59:53.686 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=started 2024-09-14T08:59:53.686 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:59:53.686 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:59:53.686 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:59:54.686 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_replay_state cluster1 mirror image_44 started 2024-09-14T08:59:54.686 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:59:54.687 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:59:54.687 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_44 2024-09-14T08:59:54.687 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=started 2024-09-14T08:59:54.687 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_result 2024-09-14T08:59:54.687 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=stopped 2024-09-14T08:59:54.688 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_44 2024-09-14T08:59:54.688 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_instance=cluster1 2024-09-14T08:59:54.688 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ grep -i state 2024-09-14T08:59:54.688 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:59:54.689 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T08:59:54.689 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance=cluster1 2024-09-14T08:59:54.689 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local loop_instance 2024-09-14T08:59:54.689 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T08:59:54.689 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sleep 0 2024-09-14T08:59:54.689 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T08:59:54.689 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ seq 0 3 2024-09-14T08:59:54.692 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:59:54.692 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_44 2024-09-14T08:59:54.692 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:0 2024-09-14T08:59:54.692 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:59:54.692 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:59:54.692 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T08:59:54.692 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:0 2024-09-14T08:59:54.692 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:59:54.692 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:59:54.692 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:59:54.692 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=0 2024-09-14T08:59:54.693 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T08:59:54.693 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:59:54.693 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:59:54.693 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=0 2024-09-14T08:59:54.693 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:59:54.693 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T08:59:54.693 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:0 2024-09-14T08:59:54.693 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:59:54.693 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:59:54.693 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T08:59:54.693 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:0 2024-09-14T08:59:54.693 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:59:54.694 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:59:54.694 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:59:54.694 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T08:59:54.694 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T08:59:54.694 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:59:54.694 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:59:54.694 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=0 2024-09-14T08:59:54.694 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=0 2024-09-14T08:59:54.694 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T08:59:54.714 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:59:54.714 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:59:54.715 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T08:59:54.715 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_44 2024-09-14T08:59:54.803 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_44'] 2024-09-14T08:59:54.803 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:59:54.803 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:59:54.806 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:59:54.806 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_44 2024-09-14T08:59:54.806 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:1 2024-09-14T08:59:54.807 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:59:54.807 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:59:54.807 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T08:59:54.807 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:1 2024-09-14T08:59:54.807 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:59:54.807 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:59:54.807 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:59:54.807 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=1 2024-09-14T08:59:54.807 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T08:59:54.807 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:59:54.807 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:59:54.807 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=1 2024-09-14T08:59:54.807 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:59:54.807 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T08:59:54.808 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:1 2024-09-14T08:59:54.808 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:59:54.808 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:59:54.808 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T08:59:54.808 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:1 2024-09-14T08:59:54.808 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:59:54.808 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:59:54.808 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:59:54.808 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T08:59:54.808 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T08:59:54.808 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:59:54.809 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:59:54.809 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=1 2024-09-14T08:59:54.809 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=1 2024-09-14T08:59:54.809 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T08:59:54.825 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:59:54.825 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:59:54.825 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T08:59:54.825 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_44 2024-09-14T08:59:54.905 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_44'] 2024-09-14T08:59:54.905 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T08:59:54.905 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T08:59:54.908 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T08:59:54.908 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_44 2024-09-14T08:59:54.908 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:2 2024-09-14T08:59:54.909 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T08:59:54.909 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T08:59:54.909 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-14T08:59:54.909 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:2 2024-09-14T08:59:54.909 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T08:59:54.909 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T08:59:54.909 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T08:59:54.909 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=2 2024-09-14T08:59:54.909 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T08:59:54.909 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T08:59:54.909 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:59:54.910 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=2 2024-09-14T08:59:54.910 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:59:54.910 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-14T08:59:54.910 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:2 2024-09-14T08:59:54.910 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T08:59:54.910 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T08:59:54.910 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T08:59:54.910 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:2 2024-09-14T08:59:54.910 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T08:59:54.910 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T08:59:54.910 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T08:59:54.910 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T08:59:54.910 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-14T08:59:54.910 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T08:59:54.910 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T08:59:54.911 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=2 2024-09-14T08:59:54.911 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=2 2024-09-14T08:59:54.911 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T08:59:54.926 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:59:54.926 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:59:54.926 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T08:59:54.926 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_44 2024-09-14T08:59:55.013 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ return 0 2024-09-14T08:59:55.014 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T08:59:55.014 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ echo ' "state": "Replaying"' 2024-09-14T08:59:55.014 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep -i Replaying 2024-09-14T08:59:55.015 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: "state": "Replaying" 2024-09-14T08:59:55.016 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_state=started 2024-09-14T08:59:55.016 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test started = started 2024-09-14T08:59:55.017 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:59:55.017 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_44 2024-09-14T08:59:55.017 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:59:55.017 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:59:55.017 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:59:55.017 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_44 2024-09-14T08:59:55.017 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = journal ']' 2024-09-14T08:59:55.017 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = snapshot ']' 2024-09-14T08:59:55.017 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_44 2024-09-14T08:59:55.018 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T08:59:55.018 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:59:55.018 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:59:55.018 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_44 2024-09-14T08:59:55.018 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster2-mirror-image_44.status 2024-09-14T08:59:55.018 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster2-mirror-image_44.status 2024-09-14T08:59:55.018 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:59:55.019 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_44.status 2024-09-14T08:59:55.019 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster1-mirror-image_44.status 2024-09-14T08:59:55.019 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster1-mirror-image_44.status 2024-09-14T08:59:55.019 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T08:59:55.019 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_status_log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_44.status 2024-09-14T08:59:55.019 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ mirror_image_snapshot cluster2 mirror image_44 2024-09-14T08:59:55.019 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:59:55.019 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:59:55.019 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_44 2024-09-14T08:59:55.020 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_44 2024-09-14T08:59:55.686 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Snapshot ID: 172 2024-09-14T08:59:55.692 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_44 /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_44.status 2024-09-14T08:59:55.692 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T08:59:55.692 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:59:55.692 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_44 2024-09-14T08:59:55.692 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_44.status 2024-09-14T08:59:55.692 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_44 --format xml 2024-09-14T08:59:55.692 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:59:55.738 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-14T08:59:55.740 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snapshot_id=172 2024-09-14T08:59:55.740 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:59:55.740 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:59:55.740 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.2 2024-09-14T08:59:55.941 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_44 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_44.status 2024-09-14T08:59:55.941 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:59:55.942 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:59:55.942 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_44 2024-09-14T08:59:55.942 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_44.status 2024-09-14T08:59:55.942 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_44 --format xml 2024-09-14T08:59:55.942 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:59:55.990 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:59:55.992 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=78 2024-09-14T08:59:55.992 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 172 = 78 2024-09-14T08:59:55.992 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:59:55.992 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.4 2024-09-14T08:59:56.393 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_44 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_44.status 2024-09-14T08:59:56.394 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:59:56.394 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:59:56.394 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_44 2024-09-14T08:59:56.394 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_44.status 2024-09-14T08:59:56.394 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_44 --format xml 2024-09-14T08:59:56.394 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:59:56.449 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:59:56.450 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:59:56.451 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:59:56.451 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:59:56.451 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:59:56.451 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:59:56.451 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 172 = '' 2024-09-14T08:59:56.452 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:59:56.452 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.8 2024-09-14T08:59:57.253 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_44 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_44.status 2024-09-14T08:59:57.253 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:59:57.253 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:59:57.253 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_44 2024-09-14T08:59:57.253 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_44.status 2024-09-14T08:59:57.253 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_44 --format xml 2024-09-14T08:59:57.253 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:59:57.307 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T08:59:57.307 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:59:57.308 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T08:59:57.308 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T08:59:57.309 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T08:59:57.309 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T08:59:57.309 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 172 = '' 2024-09-14T08:59:57.309 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T08:59:57.309 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1.6 2024-09-14T08:59:58.910 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_44 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_44.status 2024-09-14T08:59:58.911 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:59:58.911 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:59:58.911 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_44 2024-09-14T08:59:58.911 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_44.status 2024-09-14T08:59:58.911 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_44 --format xml 2024-09-14T08:59:58.911 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T08:59:58.966 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T08:59:58.968 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=172 2024-09-14T08:59:58.968 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 172 = 172 2024-09-14T08:59:58.968 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T08:59:58.968 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snap_present cluster1 mirror image_44 snap 2024-09-14T08:59:58.968 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T08:59:58.968 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T08:59:58.968 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_44 2024-09-14T08:59:58.968 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T08:59:58.968 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T08:59:58.968 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T08:59:58.969 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T08:59:59.946 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.0 2024-09-14T08:59:59.946 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Sent signal 15 2024-09-14T08:59:59.947 INFO:tasks.rbd_mirror_thrash:waiting for 9 secs before reviving daemons 2024-09-14T08:59:59.947 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi164.stderr:2024-09-14T08:59:59.947+0000 7f97ff326640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 (PID: 49690) UID: 1000 2024-09-14T08:59:59.969 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 info mirror/image_44@snap 2024-09-14T09:00:00.020 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_44': 2024-09-14T09:00:00.020 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T09:00:00.020 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T09:00:00.020 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 2 2024-09-14T09:00:00.020 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 12a02788f8d8 2024-09-14T09:00:00.020 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.12a02788f8d8 2024-09-14T09:00:00.020 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T09:00:00.020 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering, non-primary 2024-09-14T09:00:00.020 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T09:00:00.020 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T09:00:00.020 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:52:31 2024 2024-09-14T09:00:00.021 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:52:31 2024 2024-09-14T09:00:00.021 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:52:31 2024 2024-09-14T09:00:00.021 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: protected: False 2024-09-14T09:00:00.021 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T09:00:00.021 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T09:00:00.021 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: 484caad3-31ac-439e-a6d6-4a11118a3aed 2024-09-14T09:00:00.021 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: false 2024-09-14T09:00:00.024 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:00:00.024 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ compare_image_snaps mirror image_44 snap 2024-09-14T09:00:00.024 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:00:00.024 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_44 2024-09-14T09:00:00.024 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T09:00:00.024 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local ret=0 2024-09-14T09:00:00.024 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local rmt_export=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_44.export 2024-09-14T09:00:00.024 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local loc_export=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_44.export 2024-09-14T09:00:00.024 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_44.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_44.export 2024-09-14T09:00:00.025 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror export image_44@snap /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_44.export 2024-09-14T09:00:00.216 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T09:00:00.224 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 -p mirror export image_44@snap /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_44.export 2024-09-14T09:00:00.488 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T09:00:00.495 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cmp /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_44.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_44.export 2024-09-14T09:00:00.549 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_44.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_44.export 2024-09-14T09:00:00.582 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:00:00.582 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:00:00.582 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_45 2024-09-14T09:00:00.583 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_snap cluster2 mirror image_45 snap 2024-09-14T09:00:00.583 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:00:00.583 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:00:00.583 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_45 2024-09-14T09:00:00.583 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T09:00:00.583 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror snap create image_45@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T09:00:00.692 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_started cluster1 mirror image_45 2024-09-14T09:00:00.692 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:00:00.692 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:00:00.692 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_45 2024-09-14T09:00:00.692 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_state cluster1 mirror image_45 started 2024-09-14T09:00:00.692 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:00:00.692 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:00:00.692 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_45 2024-09-14T09:00:00.692 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=started 2024-09-14T09:00:00.692 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T09:00:00.693 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T09:00:00.693 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T09:00:01.692 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_replay_state cluster1 mirror image_45 started 2024-09-14T09:00:01.693 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:00:01.693 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:00:01.693 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_45 2024-09-14T09:00:01.693 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=started 2024-09-14T09:00:01.693 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_result 2024-09-14T09:00:01.693 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=stopped 2024-09-14T09:00:01.694 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_45 2024-09-14T09:00:01.694 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_instance=cluster1 2024-09-14T09:00:01.694 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ grep -i state 2024-09-14T09:00:01.694 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T09:00:01.694 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:00:01.694 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance=cluster1 2024-09-14T09:00:01.694 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local loop_instance 2024-09-14T09:00:01.694 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T09:00:01.695 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sleep 0 2024-09-14T09:00:01.695 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T09:00:01.695 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ seq 0 3 2024-09-14T09:00:01.697 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T09:00:01.697 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_45 2024-09-14T09:00:01.697 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:0 2024-09-14T09:00:01.697 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T09:00:01.697 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T09:00:01.697 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T09:00:01.697 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:0 2024-09-14T09:00:01.697 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T09:00:01.697 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T09:00:01.697 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T09:00:01.697 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=0 2024-09-14T09:00:01.698 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T09:00:01.698 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T09:00:01.698 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T09:00:01.698 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=0 2024-09-14T09:00:01.698 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T09:00:01.698 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T09:00:01.698 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:0 2024-09-14T09:00:01.698 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T09:00:01.698 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T09:00:01.698 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T09:00:01.698 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:0 2024-09-14T09:00:01.698 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T09:00:01.699 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T09:00:01.699 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T09:00:01.699 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T09:00:01.699 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T09:00:01.700 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T09:00:01.700 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T09:00:01.700 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=0 2024-09-14T09:00:01.700 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=0 2024-09-14T09:00:01.700 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T09:00:01.719 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T09:00:01.719 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T09:00:01.719 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T09:00:01.719 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_45 2024-09-14T09:00:01.806 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_45'] 2024-09-14T09:00:01.852 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T09:00:01.852 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T09:00:01.855 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T09:00:01.855 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_45 2024-09-14T09:00:01.855 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:1 2024-09-14T09:00:01.855 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T09:00:01.856 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T09:00:01.856 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T09:00:01.856 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:1 2024-09-14T09:00:01.856 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T09:00:01.856 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T09:00:01.856 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T09:00:01.856 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=1 2024-09-14T09:00:01.856 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T09:00:01.856 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T09:00:01.856 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T09:00:01.856 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=1 2024-09-14T09:00:01.856 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T09:00:01.856 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T09:00:01.857 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:1 2024-09-14T09:00:01.857 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T09:00:01.857 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T09:00:01.857 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T09:00:01.857 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:1 2024-09-14T09:00:01.857 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T09:00:01.857 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T09:00:01.857 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T09:00:01.857 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T09:00:01.857 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T09:00:01.857 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T09:00:01.857 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T09:00:01.857 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=1 2024-09-14T09:00:01.858 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=1 2024-09-14T09:00:01.858 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T09:00:01.858 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T09:00:01.858 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T09:00:01.858 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T09:00:01.858 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_45 2024-09-14T09:00:01.911 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_45'] 2024-09-14T09:00:01.912 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T09:00:01.912 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T09:00:01.914 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T09:00:01.915 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_45 2024-09-14T09:00:01.915 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:2 2024-09-14T09:00:01.915 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T09:00:01.915 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T09:00:01.915 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-14T09:00:01.915 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:2 2024-09-14T09:00:01.915 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T09:00:01.915 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T09:00:01.915 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T09:00:01.915 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=2 2024-09-14T09:00:01.915 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T09:00:01.915 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T09:00:01.915 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T09:00:01.916 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=2 2024-09-14T09:00:01.916 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T09:00:01.916 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-14T09:00:01.916 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:2 2024-09-14T09:00:01.916 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T09:00:01.916 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T09:00:01.916 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T09:00:01.916 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:2 2024-09-14T09:00:01.916 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T09:00:01.916 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T09:00:01.916 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T09:00:01.916 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T09:00:01.916 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-14T09:00:01.916 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T09:00:01.916 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T09:00:01.917 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=2 2024-09-14T09:00:01.917 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=2 2024-09-14T09:00:01.917 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T09:00:01.933 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T09:00:01.933 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T09:00:01.933 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T09:00:01.934 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_45 2024-09-14T09:00:02.023 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ return 0 2024-09-14T09:00:02.024 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T09:00:02.024 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ echo ' "state": "Replaying"' 2024-09-14T09:00:02.024 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep -i Replaying 2024-09-14T09:00:02.025 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: "state": "Replaying" 2024-09-14T09:00:02.026 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_state=started 2024-09-14T09:00:02.027 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test started = started 2024-09-14T09:00:02.027 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:00:02.027 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_45 2024-09-14T09:00:02.027 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T09:00:02.027 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:00:02.027 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:00:02.027 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_45 2024-09-14T09:00:02.027 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = journal ']' 2024-09-14T09:00:02.027 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = snapshot ']' 2024-09-14T09:00:02.027 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_45 2024-09-14T09:00:02.027 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T09:00:02.027 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:00:02.027 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:00:02.028 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_45 2024-09-14T09:00:02.028 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster2-mirror-image_45.status 2024-09-14T09:00:02.028 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster2-mirror-image_45.status 2024-09-14T09:00:02.028 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T09:00:02.029 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_45.status 2024-09-14T09:00:02.029 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster1-mirror-image_45.status 2024-09-14T09:00:02.029 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster1-mirror-image_45.status 2024-09-14T09:00:02.029 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T09:00:02.029 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_status_log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_45.status 2024-09-14T09:00:02.029 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ mirror_image_snapshot cluster2 mirror image_45 2024-09-14T09:00:02.029 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:00:02.029 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:00:02.029 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_45 2024-09-14T09:00:02.029 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_45 2024-09-14T09:00:02.710 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Snapshot ID: 174 2024-09-14T09:00:02.716 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_45 /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_45.status 2024-09-14T09:00:02.717 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:00:02.717 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:00:02.717 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_45 2024-09-14T09:00:02.717 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_45.status 2024-09-14T09:00:02.717 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_45 --format xml 2024-09-14T09:00:02.717 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T09:00:02.966 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-14T09:00:02.967 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snapshot_id=174 2024-09-14T09:00:02.967 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T09:00:02.967 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T09:00:02.967 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.2 2024-09-14T09:00:03.168 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_45 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_45.status 2024-09-14T09:00:03.169 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:00:03.169 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:00:03.169 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_45 2024-09-14T09:00:03.169 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_45.status 2024-09-14T09:00:03.169 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_45 --format xml 2024-09-14T09:00:03.169 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T09:00:03.217 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T09:00:03.219 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=79 2024-09-14T09:00:03.219 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 174 = 79 2024-09-14T09:00:03.219 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T09:00:03.219 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.4 2024-09-14T09:00:03.620 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_45 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_45.status 2024-09-14T09:00:03.620 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:00:03.620 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:00:03.621 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_45 2024-09-14T09:00:03.621 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_45.status 2024-09-14T09:00:03.621 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_45 --format xml 2024-09-14T09:00:03.621 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T09:00:03.668 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T09:00:03.669 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T09:00:03.670 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T09:00:03.670 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T09:00:03.670 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T09:00:03.670 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T09:00:03.671 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 174 = '' 2024-09-14T09:00:03.671 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T09:00:03.671 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.8 2024-09-14T09:00:04.472 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_45 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_45.status 2024-09-14T09:00:04.472 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:00:04.472 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:00:04.472 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_45 2024-09-14T09:00:04.472 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_45.status 2024-09-14T09:00:04.472 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_45 --format xml 2024-09-14T09:00:04.472 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T09:00:04.612 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T09:00:04.612 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T09:00:04.613 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T09:00:04.613 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T09:00:04.613 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T09:00:04.614 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T09:00:04.614 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 174 = '' 2024-09-14T09:00:04.614 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T09:00:04.614 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1.6 2024-09-14T09:00:06.216 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_45 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_45.status 2024-09-14T09:00:06.216 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:00:06.216 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:00:06.216 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_45 2024-09-14T09:00:06.216 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_45.status 2024-09-14T09:00:06.216 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_45 --format xml 2024-09-14T09:00:06.216 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T09:00:06.273 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T09:00:06.275 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=174 2024-09-14T09:00:06.276 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 174 = 174 2024-09-14T09:00:06.276 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:00:06.276 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snap_present cluster1 mirror image_45 snap 2024-09-14T09:00:06.276 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:00:06.276 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:00:06.276 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_45 2024-09-14T09:00:06.276 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T09:00:06.276 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T09:00:06.276 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T09:00:06.276 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T09:00:07.276 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 info mirror/image_45@snap 2024-09-14T09:00:07.324 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_45': 2024-09-14T09:00:07.324 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T09:00:07.324 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T09:00:07.324 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 2 2024-09-14T09:00:07.324 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 10bd4965d02c 2024-09-14T09:00:07.324 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.10bd4965d02c 2024-09-14T09:00:07.324 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T09:00:07.324 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering, non-primary 2024-09-14T09:00:07.324 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T09:00:07.324 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T09:00:07.325 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:52:39 2024 2024-09-14T09:00:07.325 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:52:39 2024 2024-09-14T09:00:07.325 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:52:39 2024 2024-09-14T09:00:07.325 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: protected: False 2024-09-14T09:00:07.325 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T09:00:07.325 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T09:00:07.325 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: c2a67c35-feb8-43a1-95f7-a8a88b0481e9 2024-09-14T09:00:07.325 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: false 2024-09-14T09:00:07.328 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:00:07.328 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ compare_image_snaps mirror image_45 snap 2024-09-14T09:00:07.328 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:00:07.329 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_45 2024-09-14T09:00:07.329 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T09:00:07.329 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local ret=0 2024-09-14T09:00:07.329 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local rmt_export=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_45.export 2024-09-14T09:00:07.329 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local loc_export=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_45.export 2024-09-14T09:00:07.329 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_45.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_45.export 2024-09-14T09:00:07.329 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror export image_45@snap /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_45.export 2024-09-14T09:00:07.533 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T09:00:07.540 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 -p mirror export image_45@snap /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_45.export 2024-09-14T09:00:07.781 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T09:00:07.788 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cmp /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_45.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_45.export 2024-09-14T09:00:07.842 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_45.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_45.export 2024-09-14T09:00:07.874 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:00:07.874 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:00:07.874 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_46 2024-09-14T09:00:07.874 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_snap cluster2 mirror image_46 snap 2024-09-14T09:00:07.874 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:00:07.874 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:00:07.874 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_46 2024-09-14T09:00:07.874 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T09:00:07.875 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror snap create image_46@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T09:00:08.275 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_started cluster1 mirror image_46 2024-09-14T09:00:08.275 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:00:08.275 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:00:08.275 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_46 2024-09-14T09:00:08.275 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_state cluster1 mirror image_46 started 2024-09-14T09:00:08.275 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:00:08.275 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:00:08.275 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_46 2024-09-14T09:00:08.276 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=started 2024-09-14T09:00:08.276 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T09:00:08.276 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T09:00:08.276 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T09:00:08.947 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.0 2024-09-14T09:00:08.948 INFO:teuthology.orchestra.run:waiting for 600 2024-09-14T09:00:08.948 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.0 2024-09-14T09:00:08.948 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Restarting daemon 2024-09-14T09:00:08.948 DEBUG:teuthology.orchestra.run.smithi164:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 2024-09-14T09:00:08.951 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Started 2024-09-14T09:00:08.952 INFO:tasks.rbd_mirror_thrash:waiting for 113 secs before thrashing 2024-09-14T09:00:09.277 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_replay_state cluster1 mirror image_46 started 2024-09-14T09:00:09.277 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:00:09.277 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:00:09.277 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_46 2024-09-14T09:00:09.277 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=started 2024-09-14T09:00:09.277 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_result 2024-09-14T09:00:09.277 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=stopped 2024-09-14T09:00:09.277 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_46 2024-09-14T09:00:09.277 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ grep -i state 2024-09-14T09:00:09.277 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_instance=cluster1 2024-09-14T09:00:09.277 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T09:00:09.277 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:00:09.277 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance=cluster1 2024-09-14T09:00:09.278 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local loop_instance 2024-09-14T09:00:09.278 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T09:00:09.278 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sleep 0 2024-09-14T09:00:09.280 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T09:00:09.280 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ seq 0 3 2024-09-14T09:00:09.280 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T09:00:09.280 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_46 2024-09-14T09:00:09.280 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:0 2024-09-14T09:00:09.280 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T09:00:09.280 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T09:00:09.280 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T09:00:09.280 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:0 2024-09-14T09:00:09.281 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T09:00:09.281 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T09:00:09.281 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T09:00:09.281 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=0 2024-09-14T09:00:09.281 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T09:00:09.281 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T09:00:09.281 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T09:00:09.281 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=0 2024-09-14T09:00:09.281 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T09:00:09.281 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T09:00:09.281 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:0 2024-09-14T09:00:09.281 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T09:00:09.281 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T09:00:09.281 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T09:00:09.281 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:0 2024-09-14T09:00:09.282 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T09:00:09.283 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T09:00:09.283 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T09:00:09.283 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T09:00:09.283 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T09:00:09.283 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T09:00:09.283 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T09:00:09.283 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=0 2024-09-14T09:00:09.283 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=0 2024-09-14T09:00:09.283 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T09:00:09.298 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T09:00:09.299 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T09:00:09.299 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T09:00:09.299 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_46 2024-09-14T09:00:09.381 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_46'] 2024-09-14T09:00:09.381 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T09:00:09.381 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T09:00:09.384 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T09:00:09.384 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_46 2024-09-14T09:00:09.385 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:1 2024-09-14T09:00:09.385 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T09:00:09.385 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T09:00:09.385 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T09:00:09.385 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:1 2024-09-14T09:00:09.385 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T09:00:09.385 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T09:00:09.385 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T09:00:09.385 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=1 2024-09-14T09:00:09.385 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T09:00:09.385 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T09:00:09.385 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T09:00:09.385 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=1 2024-09-14T09:00:09.386 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T09:00:09.386 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T09:00:09.386 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:1 2024-09-14T09:00:09.386 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T09:00:09.386 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T09:00:09.386 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T09:00:09.386 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:1 2024-09-14T09:00:09.386 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T09:00:09.386 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T09:00:09.386 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T09:00:09.386 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T09:00:09.386 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T09:00:09.386 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T09:00:09.386 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T09:00:09.386 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=1 2024-09-14T09:00:09.387 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=1 2024-09-14T09:00:09.387 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T09:00:09.402 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T09:00:09.402 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T09:00:09.402 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T09:00:09.403 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_46 2024-09-14T09:00:09.492 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ return 0 2024-09-14T09:00:09.492 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T09:00:09.492 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ echo ' "state": "Replaying"' 2024-09-14T09:00:09.492 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep -i Replaying 2024-09-14T09:00:09.493 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: "state": "Replaying" 2024-09-14T09:00:09.495 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_state=started 2024-09-14T09:00:09.495 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test started = started 2024-09-14T09:00:09.495 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:00:09.495 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_46 2024-09-14T09:00:09.495 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T09:00:09.495 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:00:09.495 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:00:09.495 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_46 2024-09-14T09:00:09.495 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = journal ']' 2024-09-14T09:00:09.495 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = snapshot ']' 2024-09-14T09:00:09.495 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_46 2024-09-14T09:00:09.495 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T09:00:09.495 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:00:09.496 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:00:09.496 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_46 2024-09-14T09:00:09.496 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster2-mirror-image_46.status 2024-09-14T09:00:09.496 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster2-mirror-image_46.status 2024-09-14T09:00:09.496 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T09:00:09.496 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_46.status 2024-09-14T09:00:09.496 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster1-mirror-image_46.status 2024-09-14T09:00:09.497 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster1-mirror-image_46.status 2024-09-14T09:00:09.497 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T09:00:09.497 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_status_log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_46.status 2024-09-14T09:00:09.497 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ mirror_image_snapshot cluster2 mirror image_46 2024-09-14T09:00:09.498 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:00:09.498 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:00:09.498 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_46 2024-09-14T09:00:09.498 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_46 2024-09-14T09:00:10.301 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Snapshot ID: 176 2024-09-14T09:00:10.307 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_46 /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_46.status 2024-09-14T09:00:10.308 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:00:10.308 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:00:10.308 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_46 2024-09-14T09:00:10.308 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_46.status 2024-09-14T09:00:10.308 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_46 --format xml 2024-09-14T09:00:10.308 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T09:00:10.356 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-14T09:00:10.357 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snapshot_id=176 2024-09-14T09:00:10.358 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T09:00:10.358 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T09:00:10.358 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.2 2024-09-14T09:00:10.559 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_46 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_46.status 2024-09-14T09:00:10.559 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:00:10.559 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:00:10.559 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_46 2024-09-14T09:00:10.559 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_46.status 2024-09-14T09:00:10.560 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_46 --format xml 2024-09-14T09:00:10.560 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T09:00:10.608 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T09:00:10.608 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T09:00:10.609 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T09:00:10.609 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T09:00:10.609 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T09:00:10.610 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T09:00:10.610 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 176 = '' 2024-09-14T09:00:10.610 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T09:00:10.610 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.4 2024-09-14T09:00:11.011 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_46 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_46.status 2024-09-14T09:00:11.011 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:00:11.012 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:00:11.012 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_46 2024-09-14T09:00:11.012 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_46.status 2024-09-14T09:00:11.012 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_46 --format xml 2024-09-14T09:00:11.012 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T09:00:11.060 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T09:00:11.061 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T09:00:11.062 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T09:00:11.062 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T09:00:11.062 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T09:00:11.063 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T09:00:11.063 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 176 = '' 2024-09-14T09:00:11.063 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T09:00:11.063 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.8 2024-09-14T09:00:11.864 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_46 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_46.status 2024-09-14T09:00:11.864 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:00:11.864 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:00:11.864 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_46 2024-09-14T09:00:11.864 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_46.status 2024-09-14T09:00:11.864 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_46 --format xml 2024-09-14T09:00:11.864 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T09:00:11.912 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T09:00:11.914 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=176 2024-09-14T09:00:11.914 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 176 = 176 2024-09-14T09:00:11.914 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:00:11.914 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snap_present cluster1 mirror image_46 snap 2024-09-14T09:00:11.914 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:00:11.914 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:00:11.914 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_46 2024-09-14T09:00:11.914 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T09:00:11.915 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T09:00:11.915 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T09:00:11.915 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T09:00:12.915 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 info mirror/image_46@snap 2024-09-14T09:00:12.960 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_46': 2024-09-14T09:00:12.961 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T09:00:12.961 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T09:00:12.961 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 2 2024-09-14T09:00:12.961 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 11895e265c70 2024-09-14T09:00:12.961 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.11895e265c70 2024-09-14T09:00:12.961 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T09:00:12.961 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering, non-primary 2024-09-14T09:00:12.961 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T09:00:12.961 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T09:00:12.961 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:52:47 2024 2024-09-14T09:00:12.961 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:52:47 2024 2024-09-14T09:00:12.961 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:52:47 2024 2024-09-14T09:00:12.961 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: protected: False 2024-09-14T09:00:12.962 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T09:00:12.962 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T09:00:12.962 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: 0f89d25d-4adb-4110-bda2-74b7704ad44b 2024-09-14T09:00:12.962 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: false 2024-09-14T09:00:12.964 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:00:12.964 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ compare_image_snaps mirror image_46 snap 2024-09-14T09:00:12.965 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:00:12.965 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_46 2024-09-14T09:00:12.965 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T09:00:12.965 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local ret=0 2024-09-14T09:00:12.965 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local rmt_export=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_46.export 2024-09-14T09:00:12.965 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local loc_export=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_46.export 2024-09-14T09:00:12.965 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_46.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_46.export 2024-09-14T09:00:12.965 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror export image_46@snap /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_46.export 2024-09-14T09:00:13.161 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T09:00:13.167 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 -p mirror export image_46@snap /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_46.export 2024-09-14T09:00:13.416 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T09:00:13.423 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cmp /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_46.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_46.export 2024-09-14T09:00:13.477 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_46.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_46.export 2024-09-14T09:00:13.508 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:00:13.508 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:00:13.509 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_47 2024-09-14T09:00:13.509 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_snap cluster2 mirror image_47 snap 2024-09-14T09:00:13.509 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:00:13.509 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:00:13.509 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_47 2024-09-14T09:00:13.509 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T09:00:13.509 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror snap create image_47@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T09:00:14.309 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_started cluster1 mirror image_47 2024-09-14T09:00:14.310 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:00:14.310 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:00:14.310 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_47 2024-09-14T09:00:14.310 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_state cluster1 mirror image_47 started 2024-09-14T09:00:14.310 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:00:14.310 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:00:14.310 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_47 2024-09-14T09:00:14.310 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=started 2024-09-14T09:00:14.310 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T09:00:14.310 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T09:00:14.310 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T09:00:15.311 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_replay_state cluster1 mirror image_47 started 2024-09-14T09:00:15.311 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:00:15.311 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:00:15.311 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_47 2024-09-14T09:00:15.311 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=started 2024-09-14T09:00:15.311 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_result 2024-09-14T09:00:15.311 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=stopped 2024-09-14T09:00:15.312 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_47 2024-09-14T09:00:15.312 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_instance=cluster1 2024-09-14T09:00:15.312 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ grep -i state 2024-09-14T09:00:15.312 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T09:00:15.312 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:00:15.312 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance=cluster1 2024-09-14T09:00:15.313 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local loop_instance 2024-09-14T09:00:15.313 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T09:00:15.313 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sleep 0 2024-09-14T09:00:15.313 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T09:00:15.313 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ seq 0 3 2024-09-14T09:00:15.316 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T09:00:15.316 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_47 2024-09-14T09:00:15.316 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:0 2024-09-14T09:00:15.316 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T09:00:15.316 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T09:00:15.316 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T09:00:15.316 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:0 2024-09-14T09:00:15.316 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T09:00:15.316 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T09:00:15.316 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T09:00:15.316 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=0 2024-09-14T09:00:15.316 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T09:00:15.316 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T09:00:15.316 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T09:00:15.317 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=0 2024-09-14T09:00:15.317 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T09:00:15.317 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T09:00:15.317 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:0 2024-09-14T09:00:15.317 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T09:00:15.317 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T09:00:15.317 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T09:00:15.317 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:0 2024-09-14T09:00:15.317 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T09:00:15.317 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T09:00:15.317 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T09:00:15.317 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T09:00:15.317 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T09:00:15.317 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T09:00:15.317 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T09:00:15.317 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=0 2024-09-14T09:00:15.318 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=0 2024-09-14T09:00:15.318 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T09:00:15.337 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T09:00:15.337 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T09:00:15.337 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T09:00:15.337 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_47 2024-09-14T09:00:15.425 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_47'] 2024-09-14T09:00:15.426 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T09:00:15.426 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T09:00:15.429 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T09:00:15.429 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_47 2024-09-14T09:00:15.429 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:1 2024-09-14T09:00:15.429 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T09:00:15.429 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T09:00:15.429 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T09:00:15.429 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:1 2024-09-14T09:00:15.429 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T09:00:15.429 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T09:00:15.429 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T09:00:15.429 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=1 2024-09-14T09:00:15.429 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T09:00:15.429 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T09:00:15.430 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T09:00:15.430 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=1 2024-09-14T09:00:15.430 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T09:00:15.430 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T09:00:15.430 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:1 2024-09-14T09:00:15.430 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T09:00:15.430 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T09:00:15.430 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T09:00:15.430 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:1 2024-09-14T09:00:15.430 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T09:00:15.430 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T09:00:15.430 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T09:00:15.430 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T09:00:15.431 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T09:00:15.431 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T09:00:15.431 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T09:00:15.431 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=1 2024-09-14T09:00:15.431 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=1 2024-09-14T09:00:15.431 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T09:00:15.447 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T09:00:15.447 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T09:00:15.447 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T09:00:15.447 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_47 2024-09-14T09:00:15.535 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ return 0 2024-09-14T09:00:15.536 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T09:00:15.536 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ echo ' "state": "Replaying"' 2024-09-14T09:00:15.536 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep -i Replaying 2024-09-14T09:00:15.537 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: "state": "Replaying" 2024-09-14T09:00:15.539 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_state=started 2024-09-14T09:00:15.539 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test started = started 2024-09-14T09:00:15.539 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:00:15.539 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_47 2024-09-14T09:00:15.539 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T09:00:15.539 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:00:15.539 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:00:15.539 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_47 2024-09-14T09:00:15.539 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = journal ']' 2024-09-14T09:00:15.539 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = snapshot ']' 2024-09-14T09:00:15.539 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_47 2024-09-14T09:00:15.539 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T09:00:15.539 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:00:15.540 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:00:15.540 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_47 2024-09-14T09:00:15.540 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster2-mirror-image_47.status 2024-09-14T09:00:15.540 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster2-mirror-image_47.status 2024-09-14T09:00:15.540 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T09:00:15.541 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_47.status 2024-09-14T09:00:15.541 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster1-mirror-image_47.status 2024-09-14T09:00:15.541 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster1-mirror-image_47.status 2024-09-14T09:00:15.541 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T09:00:15.541 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_status_log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_47.status 2024-09-14T09:00:15.541 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ mirror_image_snapshot cluster2 mirror image_47 2024-09-14T09:00:15.541 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:00:15.541 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:00:15.541 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_47 2024-09-14T09:00:15.541 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_47 2024-09-14T09:00:16.320 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Snapshot ID: 178 2024-09-14T09:00:16.327 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_47 /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_47.status 2024-09-14T09:00:16.327 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:00:16.327 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:00:16.327 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_47 2024-09-14T09:00:16.327 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_47.status 2024-09-14T09:00:16.327 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_47 --format xml 2024-09-14T09:00:16.327 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T09:00:16.374 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-14T09:00:16.376 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snapshot_id=178 2024-09-14T09:00:16.376 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T09:00:16.376 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T09:00:16.376 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.2 2024-09-14T09:00:16.577 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_47 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_47.status 2024-09-14T09:00:16.578 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:00:16.578 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:00:16.578 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_47 2024-09-14T09:00:16.578 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_47.status 2024-09-14T09:00:16.578 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_47 --format xml 2024-09-14T09:00:16.578 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T09:00:16.626 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T09:00:16.626 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T09:00:16.627 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T09:00:16.627 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T09:00:16.627 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T09:00:16.628 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T09:00:16.628 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 178 = '' 2024-09-14T09:00:16.628 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T09:00:16.628 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.4 2024-09-14T09:00:17.029 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_47 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_47.status 2024-09-14T09:00:17.029 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:00:17.030 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:00:17.030 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_47 2024-09-14T09:00:17.030 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_47.status 2024-09-14T09:00:17.030 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_47 --format xml 2024-09-14T09:00:17.030 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T09:00:17.079 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T09:00:17.079 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T09:00:17.080 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T09:00:17.080 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T09:00:17.080 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T09:00:17.081 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T09:00:17.081 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 178 = '' 2024-09-14T09:00:17.081 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T09:00:17.081 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.8 2024-09-14T09:00:17.882 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_47 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_47.status 2024-09-14T09:00:17.882 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:00:17.882 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:00:17.882 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_47 2024-09-14T09:00:17.882 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_47.status 2024-09-14T09:00:17.882 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_47 --format xml 2024-09-14T09:00:17.882 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T09:00:17.930 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T09:00:17.932 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=178 2024-09-14T09:00:17.932 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 178 = 178 2024-09-14T09:00:17.932 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:00:17.932 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snap_present cluster1 mirror image_47 snap 2024-09-14T09:00:17.933 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:00:17.933 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:00:17.933 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_47 2024-09-14T09:00:17.933 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T09:00:17.933 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T09:00:17.933 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T09:00:17.933 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T09:00:18.933 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 info mirror/image_47@snap 2024-09-14T09:00:18.977 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_47': 2024-09-14T09:00:18.977 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T09:00:18.977 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T09:00:18.977 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 2 2024-09-14T09:00:18.977 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 12966fd65413 2024-09-14T09:00:18.977 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.12966fd65413 2024-09-14T09:00:18.977 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T09:00:18.977 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering, non-primary 2024-09-14T09:00:18.977 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T09:00:18.978 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T09:00:18.978 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:52:55 2024 2024-09-14T09:00:18.978 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:52:55 2024 2024-09-14T09:00:18.978 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:52:55 2024 2024-09-14T09:00:18.978 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: protected: False 2024-09-14T09:00:18.978 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T09:00:18.978 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T09:00:18.978 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: 0654f4ee-c91c-4318-964b-9a7c32deb872 2024-09-14T09:00:18.978 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: false 2024-09-14T09:00:18.981 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:00:18.981 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ compare_image_snaps mirror image_47 snap 2024-09-14T09:00:18.981 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:00:18.981 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_47 2024-09-14T09:00:18.981 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T09:00:18.981 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local ret=0 2024-09-14T09:00:18.981 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local rmt_export=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_47.export 2024-09-14T09:00:18.981 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local loc_export=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_47.export 2024-09-14T09:00:18.981 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_47.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_47.export 2024-09-14T09:00:18.982 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror export image_47@snap /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_47.export 2024-09-14T09:00:19.171 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T09:00:19.178 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 -p mirror export image_47@snap /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_47.export 2024-09-14T09:00:19.454 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T09:00:19.462 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cmp /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_47.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_47.export 2024-09-14T09:00:19.516 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_47.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_47.export 2024-09-14T09:00:19.549 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:00:19.549 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:00:19.549 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_48 2024-09-14T09:00:19.549 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_snap cluster2 mirror image_48 snap 2024-09-14T09:00:19.549 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:00:19.549 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:00:19.549 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_48 2024-09-14T09:00:19.549 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T09:00:19.549 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror snap create image_48@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T09:00:20.325 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_started cluster1 mirror image_48 2024-09-14T09:00:20.325 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:00:20.325 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:00:20.325 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_48 2024-09-14T09:00:20.325 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_state cluster1 mirror image_48 started 2024-09-14T09:00:20.325 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:00:20.325 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:00:20.325 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_48 2024-09-14T09:00:20.325 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=started 2024-09-14T09:00:20.325 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T09:00:20.325 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T09:00:20.325 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T09:00:21.326 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_replay_state cluster1 mirror image_48 started 2024-09-14T09:00:21.326 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:00:21.326 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:00:21.326 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_48 2024-09-14T09:00:21.326 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=started 2024-09-14T09:00:21.326 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_result 2024-09-14T09:00:21.326 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=stopped 2024-09-14T09:00:21.327 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_48 2024-09-14T09:00:21.327 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_instance=cluster1 2024-09-14T09:00:21.327 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T09:00:21.327 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ grep -i state 2024-09-14T09:00:21.327 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:00:21.327 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance=cluster1 2024-09-14T09:00:21.328 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local loop_instance 2024-09-14T09:00:21.328 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T09:00:21.328 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sleep 0 2024-09-14T09:00:21.328 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T09:00:21.330 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ seq 0 3 2024-09-14T09:00:21.331 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T09:00:21.331 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_48 2024-09-14T09:00:21.331 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:0 2024-09-14T09:00:21.331 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T09:00:21.331 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T09:00:21.331 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T09:00:21.331 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:0 2024-09-14T09:00:21.331 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T09:00:21.331 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T09:00:21.331 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T09:00:21.331 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=0 2024-09-14T09:00:21.331 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T09:00:21.331 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T09:00:21.332 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T09:00:21.332 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=0 2024-09-14T09:00:21.332 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T09:00:21.332 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T09:00:21.332 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:0 2024-09-14T09:00:21.332 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T09:00:21.332 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T09:00:21.332 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T09:00:21.332 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:0 2024-09-14T09:00:21.333 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T09:00:21.333 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T09:00:21.333 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T09:00:21.333 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T09:00:21.333 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T09:00:21.333 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T09:00:21.333 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T09:00:21.333 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=0 2024-09-14T09:00:21.333 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=0 2024-09-14T09:00:21.333 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T09:00:21.352 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T09:00:21.353 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T09:00:21.353 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T09:00:21.353 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_48 2024-09-14T09:00:21.441 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_48'] 2024-09-14T09:00:21.442 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T09:00:21.442 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T09:00:21.445 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T09:00:21.445 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_48 2024-09-14T09:00:21.445 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:1 2024-09-14T09:00:21.445 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T09:00:21.445 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T09:00:21.446 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T09:00:21.446 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:1 2024-09-14T09:00:21.446 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T09:00:21.446 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T09:00:21.446 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T09:00:21.446 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=1 2024-09-14T09:00:21.446 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T09:00:21.446 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T09:00:21.446 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T09:00:21.446 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=1 2024-09-14T09:00:21.446 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T09:00:21.446 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T09:00:21.446 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:1 2024-09-14T09:00:21.446 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T09:00:21.447 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T09:00:21.447 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T09:00:21.447 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:1 2024-09-14T09:00:21.447 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T09:00:21.447 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T09:00:21.447 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T09:00:21.447 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T09:00:21.447 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T09:00:21.447 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T09:00:21.448 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T09:00:21.448 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=1 2024-09-14T09:00:21.448 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=1 2024-09-14T09:00:21.448 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T09:00:21.463 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T09:00:21.463 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T09:00:21.463 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T09:00:21.463 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_48 2024-09-14T09:00:21.545 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_48'] 2024-09-14T09:00:21.545 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T09:00:21.546 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T09:00:21.548 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T09:00:21.548 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_48 2024-09-14T09:00:21.548 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:2 2024-09-14T09:00:21.549 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T09:00:21.549 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T09:00:21.549 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-14T09:00:21.549 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:2 2024-09-14T09:00:21.549 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T09:00:21.549 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T09:00:21.549 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T09:00:21.549 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=2 2024-09-14T09:00:21.549 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T09:00:21.549 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T09:00:21.549 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T09:00:21.549 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=2 2024-09-14T09:00:21.549 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T09:00:21.549 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-14T09:00:21.550 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:2 2024-09-14T09:00:21.550 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T09:00:21.550 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T09:00:21.550 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T09:00:21.550 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:2 2024-09-14T09:00:21.550 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T09:00:21.550 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T09:00:21.550 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T09:00:21.550 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T09:00:21.550 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-14T09:00:21.550 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T09:00:21.550 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T09:00:21.550 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=2 2024-09-14T09:00:21.550 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=2 2024-09-14T09:00:21.550 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T09:00:21.567 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T09:00:21.567 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T09:00:21.567 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T09:00:21.567 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_48 2024-09-14T09:00:21.657 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ return 0 2024-09-14T09:00:21.657 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T09:00:21.657 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ echo ' "state": "Replaying"' 2024-09-14T09:00:21.658 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep -i Replaying 2024-09-14T09:00:21.658 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: "state": "Replaying" 2024-09-14T09:00:21.659 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_state=started 2024-09-14T09:00:21.660 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test started = started 2024-09-14T09:00:21.660 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:00:21.660 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_48 2024-09-14T09:00:21.660 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T09:00:21.660 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:00:21.660 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:00:21.660 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_48 2024-09-14T09:00:21.660 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = journal ']' 2024-09-14T09:00:21.660 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = snapshot ']' 2024-09-14T09:00:21.660 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_48 2024-09-14T09:00:21.660 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T09:00:21.660 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:00:21.660 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:00:21.661 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_48 2024-09-14T09:00:21.661 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster2-mirror-image_48.status 2024-09-14T09:00:21.661 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster2-mirror-image_48.status 2024-09-14T09:00:21.661 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T09:00:21.662 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_48.status 2024-09-14T09:00:21.662 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster1-mirror-image_48.status 2024-09-14T09:00:21.662 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster1-mirror-image_48.status 2024-09-14T09:00:21.662 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T09:00:21.662 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_status_log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_48.status 2024-09-14T09:00:21.662 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ mirror_image_snapshot cluster2 mirror image_48 2024-09-14T09:00:21.662 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:00:21.662 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:00:21.662 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_48 2024-09-14T09:00:21.663 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_48 2024-09-14T09:00:22.339 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Snapshot ID: 180 2024-09-14T09:00:22.345 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_48 /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_48.status 2024-09-14T09:00:22.345 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:00:22.345 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:00:22.345 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_48 2024-09-14T09:00:22.346 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_48.status 2024-09-14T09:00:22.346 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_48 --format xml 2024-09-14T09:00:22.346 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T09:00:22.393 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-14T09:00:22.394 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snapshot_id=180 2024-09-14T09:00:22.395 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T09:00:22.395 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T09:00:22.395 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.2 2024-09-14T09:00:22.596 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_48 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_48.status 2024-09-14T09:00:22.596 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:00:22.596 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:00:22.596 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_48 2024-09-14T09:00:22.596 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_48.status 2024-09-14T09:00:22.596 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_48 --format xml 2024-09-14T09:00:22.596 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T09:00:22.644 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T09:00:22.644 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T09:00:22.646 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T09:00:22.646 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T09:00:22.646 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T09:00:22.646 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T09:00:22.646 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 180 = '' 2024-09-14T09:00:22.646 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T09:00:22.646 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.4 2024-09-14T09:00:23.047 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_48 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_48.status 2024-09-14T09:00:23.047 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:00:23.047 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:00:23.047 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_48 2024-09-14T09:00:23.047 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_48.status 2024-09-14T09:00:23.047 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_48 --format xml 2024-09-14T09:00:23.047 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T09:00:23.097 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T09:00:23.097 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T09:00:23.099 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T09:00:23.099 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T09:00:23.099 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T09:00:23.099 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T09:00:23.099 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 180 = '' 2024-09-14T09:00:23.099 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T09:00:23.099 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.8 2024-09-14T09:00:23.901 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_48 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_48.status 2024-09-14T09:00:23.901 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:00:23.901 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:00:23.901 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_48 2024-09-14T09:00:23.901 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_48.status 2024-09-14T09:00:23.901 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_48 --format xml 2024-09-14T09:00:23.901 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T09:00:23.949 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T09:00:23.951 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=180 2024-09-14T09:00:23.951 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 180 = 180 2024-09-14T09:00:23.951 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:00:23.951 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snap_present cluster1 mirror image_48 snap 2024-09-14T09:00:23.951 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:00:23.951 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:00:23.951 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_48 2024-09-14T09:00:23.951 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T09:00:23.951 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T09:00:23.951 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T09:00:23.951 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T09:00:24.952 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 info mirror/image_48@snap 2024-09-14T09:00:25.000 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_48': 2024-09-14T09:00:25.000 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T09:00:25.000 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T09:00:25.000 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 2 2024-09-14T09:00:25.000 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 12a0678cda06 2024-09-14T09:00:25.000 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.12a0678cda06 2024-09-14T09:00:25.000 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T09:00:25.000 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering, non-primary 2024-09-14T09:00:25.000 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T09:00:25.000 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T09:00:25.000 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:53:03 2024 2024-09-14T09:00:25.001 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:53:03 2024 2024-09-14T09:00:25.001 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:53:03 2024 2024-09-14T09:00:25.001 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: protected: False 2024-09-14T09:00:25.001 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T09:00:25.001 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T09:00:25.001 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: d56603da-84e8-4df7-8c5e-01193a0d3227 2024-09-14T09:00:25.001 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: false 2024-09-14T09:00:25.004 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:00:25.004 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ compare_image_snaps mirror image_48 snap 2024-09-14T09:00:25.004 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:00:25.004 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_48 2024-09-14T09:00:25.004 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T09:00:25.004 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local ret=0 2024-09-14T09:00:25.004 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local rmt_export=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_48.export 2024-09-14T09:00:25.004 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local loc_export=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_48.export 2024-09-14T09:00:25.004 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_48.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_48.export 2024-09-14T09:00:25.005 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror export image_48@snap /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_48.export 2024-09-14T09:00:25.209 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T09:00:25.217 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 -p mirror export image_48@snap /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_48.export 2024-09-14T09:00:25.439 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T09:00:25.446 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cmp /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_48.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_48.export 2024-09-14T09:00:25.500 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_48.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_48.export 2024-09-14T09:00:25.532 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:00:25.532 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:00:25.533 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_49 2024-09-14T09:00:25.533 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_snap cluster2 mirror image_49 snap 2024-09-14T09:00:25.533 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:00:25.533 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:00:25.533 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_49 2024-09-14T09:00:25.533 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T09:00:25.533 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror snap create image_49@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T09:00:26.171 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.52218771366696) > weight (0.25) 2024-09-14T09:00:26.171 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.7515166089939772) > weight (0.25) 2024-09-14T09:00:26.171 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.2 2024-09-14T09:00:26.171 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Sent signal 15 2024-09-14T09:00:26.172 INFO:tasks.rbd_mirror_thrash:waiting for 4 secs before reviving daemons 2024-09-14T09:00:26.172 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi164.stderr:2024-09-14T09:00:26.172+0000 7fbaf9022640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 (PID: 36814) UID: 1000 2024-09-14T09:00:26.587 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_started cluster1 mirror image_49 2024-09-14T09:00:26.587 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:00:26.587 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:00:26.587 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_49 2024-09-14T09:00:26.587 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_state cluster1 mirror image_49 started 2024-09-14T09:00:26.587 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:00:26.587 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:00:26.587 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_49 2024-09-14T09:00:26.588 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=started 2024-09-14T09:00:26.588 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T09:00:26.588 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T09:00:26.588 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T09:00:27.589 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_replay_state cluster1 mirror image_49 started 2024-09-14T09:00:27.589 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:00:27.589 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:00:27.589 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_49 2024-09-14T09:00:27.589 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=started 2024-09-14T09:00:27.589 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_result 2024-09-14T09:00:27.589 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=stopped 2024-09-14T09:00:27.589 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_49 2024-09-14T09:00:27.589 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_instance=cluster1 2024-09-14T09:00:27.589 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T09:00:27.589 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:00:27.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance=cluster1 2024-09-14T09:00:27.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ grep -i state 2024-09-14T09:00:27.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local loop_instance 2024-09-14T09:00:27.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T09:00:27.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sleep 0 2024-09-14T09:00:27.593 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T09:00:27.593 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ seq 0 3 2024-09-14T09:00:27.593 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T09:00:27.593 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_49 2024-09-14T09:00:27.593 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:0 2024-09-14T09:00:27.593 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T09:00:27.593 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T09:00:27.593 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T09:00:27.593 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:0 2024-09-14T09:00:27.593 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T09:00:27.593 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T09:00:27.593 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T09:00:27.594 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=0 2024-09-14T09:00:27.594 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T09:00:27.594 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T09:00:27.594 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T09:00:27.594 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=0 2024-09-14T09:00:27.594 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T09:00:27.594 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T09:00:27.594 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:0 2024-09-14T09:00:27.594 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T09:00:27.594 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T09:00:27.594 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T09:00:27.594 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:0 2024-09-14T09:00:27.594 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T09:00:27.594 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T09:00:27.595 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T09:00:27.595 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T09:00:27.595 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T09:00:27.595 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T09:00:27.595 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T09:00:27.595 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=0 2024-09-14T09:00:27.595 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=0 2024-09-14T09:00:27.595 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T09:00:27.613 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T09:00:27.613 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T09:00:27.613 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T09:00:27.613 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_49 2024-09-14T09:00:27.697 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_49'] 2024-09-14T09:00:27.698 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T09:00:27.698 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T09:00:27.701 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T09:00:27.701 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_49 2024-09-14T09:00:27.701 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:1 2024-09-14T09:00:27.701 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T09:00:27.701 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T09:00:27.701 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T09:00:27.701 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:1 2024-09-14T09:00:27.701 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T09:00:27.702 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T09:00:27.702 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T09:00:27.702 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=1 2024-09-14T09:00:27.702 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T09:00:27.702 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T09:00:27.702 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T09:00:27.702 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=1 2024-09-14T09:00:27.702 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T09:00:27.702 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T09:00:27.702 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:1 2024-09-14T09:00:27.702 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T09:00:27.702 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T09:00:27.702 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T09:00:27.702 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:1 2024-09-14T09:00:27.703 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T09:00:27.703 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T09:00:27.703 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T09:00:27.703 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T09:00:27.703 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T09:00:27.703 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T09:00:27.703 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T09:00:27.703 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=1 2024-09-14T09:00:27.703 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=1 2024-09-14T09:00:27.703 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T09:00:27.719 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T09:00:27.719 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T09:00:27.720 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T09:00:27.720 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_49 2024-09-14T09:00:27.800 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_49'] 2024-09-14T09:00:27.801 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T09:00:27.801 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T09:00:27.803 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T09:00:27.804 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_49 2024-09-14T09:00:27.804 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:2 2024-09-14T09:00:27.804 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T09:00:27.804 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T09:00:27.804 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-14T09:00:27.804 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:2 2024-09-14T09:00:27.804 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T09:00:27.804 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T09:00:27.804 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T09:00:27.804 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=2 2024-09-14T09:00:27.804 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T09:00:27.804 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T09:00:27.804 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T09:00:27.805 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=2 2024-09-14T09:00:27.805 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T09:00:27.805 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-14T09:00:27.805 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:2 2024-09-14T09:00:27.805 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T09:00:27.805 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T09:00:27.805 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T09:00:27.805 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:2 2024-09-14T09:00:27.805 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T09:00:27.805 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T09:00:27.805 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T09:00:27.805 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T09:00:27.805 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-14T09:00:27.806 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T09:00:27.806 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T09:00:27.806 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=2 2024-09-14T09:00:27.806 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=2 2024-09-14T09:00:27.806 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T09:00:27.822 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T09:00:27.823 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T09:00:27.823 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T09:00:27.823 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_49 2024-09-14T09:00:27.887 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2024-09-14T09:00:27.891 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T09:00:27.891 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_49 2024-09-14T09:00:27.891 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:3 2024-09-14T09:00:27.891 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T09:00:27.891 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T09:00:27.891 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:3 cluster instance 2024-09-14T09:00:27.891 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:3 2024-09-14T09:00:27.891 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T09:00:27.891 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T09:00:27.891 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T09:00:27.892 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=3 2024-09-14T09:00:27.892 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-14T09:00:27.892 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T09:00:27.892 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T09:00:27.892 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=3 2024-09-14T09:00:27.892 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=3 2024-09-14T09:00:27.892 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:3 cluster1 2024-09-14T09:00:27.892 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:3 2024-09-14T09:00:27.892 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T09:00:27.892 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T09:00:27.892 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-14T09:00:27.892 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:3 2024-09-14T09:00:27.892 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T09:00:27.892 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T09:00:27.893 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T09:00:27.893 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=3 2024-09-14T09:00:27.893 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 3 = cluster1:3 ']' 2024-09-14T09:00:27.893 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T09:00:27.893 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T09:00:27.893 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=3 2024-09-14T09:00:27.893 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=3 2024-09-14T09:00:27.893 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-14T09:00:27.913 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T09:00:27.913 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T09:00:27.913 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T09:00:27.913 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_49 2024-09-14T09:00:27.991 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_49'] 2024-09-14T09:00:27.991 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T09:00:27.991 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T09:00:27.992 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T09:00:27.992 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sleep 1 2024-09-14T09:00:28.994 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T09:00:28.994 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ seq 0 3 2024-09-14T09:00:28.997 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T09:00:28.997 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_49 2024-09-14T09:00:28.997 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:0 2024-09-14T09:00:28.997 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T09:00:28.997 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T09:00:28.997 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T09:00:28.997 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:0 2024-09-14T09:00:28.998 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T09:00:28.998 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T09:00:28.998 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T09:00:28.998 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=0 2024-09-14T09:00:28.998 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T09:00:28.998 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T09:00:28.998 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T09:00:28.998 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=0 2024-09-14T09:00:28.998 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T09:00:28.998 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T09:00:28.998 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:0 2024-09-14T09:00:28.998 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T09:00:28.998 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T09:00:28.999 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T09:00:28.999 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:0 2024-09-14T09:00:28.999 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T09:00:28.999 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T09:00:28.999 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T09:00:28.999 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T09:00:28.999 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T09:00:28.999 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T09:00:28.999 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T09:00:28.999 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=0 2024-09-14T09:00:28.999 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=0 2024-09-14T09:00:28.999 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T09:00:29.017 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T09:00:29.017 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T09:00:29.017 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T09:00:29.018 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_49 2024-09-14T09:00:29.105 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_49'] 2024-09-14T09:00:29.105 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T09:00:29.105 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T09:00:29.108 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T09:00:29.108 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_49 2024-09-14T09:00:29.108 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:1 2024-09-14T09:00:29.108 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T09:00:29.108 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T09:00:29.108 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T09:00:29.108 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:1 2024-09-14T09:00:29.108 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T09:00:29.108 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T09:00:29.108 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T09:00:29.108 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=1 2024-09-14T09:00:29.108 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T09:00:29.109 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T09:00:29.109 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T09:00:29.109 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=1 2024-09-14T09:00:29.109 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T09:00:29.109 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T09:00:29.109 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:1 2024-09-14T09:00:29.109 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T09:00:29.109 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T09:00:29.109 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T09:00:29.109 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:1 2024-09-14T09:00:29.109 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T09:00:29.109 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T09:00:29.109 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T09:00:29.109 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T09:00:29.109 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T09:00:29.109 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T09:00:29.109 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T09:00:29.110 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=1 2024-09-14T09:00:29.110 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=1 2024-09-14T09:00:29.110 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T09:00:29.127 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T09:00:29.127 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T09:00:29.127 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T09:00:29.127 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_49 2024-09-14T09:00:29.210 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_49'] 2024-09-14T09:00:29.210 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T09:00:29.210 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T09:00:29.213 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T09:00:29.213 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_49 2024-09-14T09:00:29.213 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:2 2024-09-14T09:00:29.213 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T09:00:29.213 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T09:00:29.213 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-14T09:00:29.213 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:2 2024-09-14T09:00:29.213 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T09:00:29.213 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T09:00:29.213 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T09:00:29.213 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=2 2024-09-14T09:00:29.214 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T09:00:29.214 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T09:00:29.214 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T09:00:29.214 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=2 2024-09-14T09:00:29.214 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T09:00:29.214 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-14T09:00:29.214 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:2 2024-09-14T09:00:29.214 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T09:00:29.214 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T09:00:29.214 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T09:00:29.214 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:2 2024-09-14T09:00:29.214 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T09:00:29.214 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T09:00:29.214 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T09:00:29.214 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T09:00:29.215 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-14T09:00:29.215 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T09:00:29.215 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T09:00:29.215 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=2 2024-09-14T09:00:29.215 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=2 2024-09-14T09:00:29.215 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T09:00:29.232 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T09:00:29.232 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T09:00:29.232 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T09:00:29.232 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_49 2024-09-14T09:00:29.296 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2024-09-14T09:00:29.300 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T09:00:29.300 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_49 2024-09-14T09:00:29.300 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:3 2024-09-14T09:00:29.300 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T09:00:29.300 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T09:00:29.300 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:3 cluster instance 2024-09-14T09:00:29.300 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:3 2024-09-14T09:00:29.300 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T09:00:29.301 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T09:00:29.301 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T09:00:29.301 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=3 2024-09-14T09:00:29.301 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-14T09:00:29.301 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T09:00:29.301 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T09:00:29.301 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=3 2024-09-14T09:00:29.301 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=3 2024-09-14T09:00:29.301 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:3 cluster1 2024-09-14T09:00:29.301 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:3 2024-09-14T09:00:29.301 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T09:00:29.301 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T09:00:29.301 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-14T09:00:29.301 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:3 2024-09-14T09:00:29.301 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T09:00:29.302 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T09:00:29.302 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T09:00:29.302 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=3 2024-09-14T09:00:29.302 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 3 = cluster1:3 ']' 2024-09-14T09:00:29.302 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T09:00:29.302 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T09:00:29.302 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=3 2024-09-14T09:00:29.302 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=3 2024-09-14T09:00:29.302 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-14T09:00:29.318 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T09:00:29.318 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T09:00:29.318 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T09:00:29.318 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_49 2024-09-14T09:00:29.400 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_49'] 2024-09-14T09:00:29.400 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T09:00:29.400 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T09:00:29.401 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T09:00:29.401 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sleep 2 2024-09-14T09:00:30.173 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.2 2024-09-14T09:00:30.173 INFO:teuthology.orchestra.run:waiting for 600 2024-09-14T09:00:30.173 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.2 2024-09-14T09:00:30.173 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Restarting daemon 2024-09-14T09:00:30.174 DEBUG:teuthology.orchestra.run.smithi164:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 2024-09-14T09:00:30.177 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Started 2024-09-14T09:00:30.177 INFO:tasks.rbd_mirror_thrash:waiting for 76 secs before thrashing 2024-09-14T09:00:31.402 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T09:00:31.403 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ seq 0 3 2024-09-14T09:00:31.406 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T09:00:31.406 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_49 2024-09-14T09:00:31.406 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:0 2024-09-14T09:00:31.406 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T09:00:31.407 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T09:00:31.407 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T09:00:31.407 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:0 2024-09-14T09:00:31.407 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T09:00:31.407 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T09:00:31.407 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T09:00:31.407 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=0 2024-09-14T09:00:31.407 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T09:00:31.407 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T09:00:31.407 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T09:00:31.407 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=0 2024-09-14T09:00:31.407 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T09:00:31.407 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T09:00:31.407 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:0 2024-09-14T09:00:31.407 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T09:00:31.408 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T09:00:31.408 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T09:00:31.408 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:0 2024-09-14T09:00:31.408 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T09:00:31.408 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T09:00:31.408 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T09:00:31.408 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T09:00:31.408 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T09:00:31.408 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T09:00:31.408 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T09:00:31.408 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=0 2024-09-14T09:00:31.408 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=0 2024-09-14T09:00:31.408 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T09:00:31.428 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T09:00:31.428 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T09:00:31.428 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T09:00:31.428 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_49 2024-09-14T09:00:31.518 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_49'] 2024-09-14T09:00:31.518 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T09:00:31.518 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T09:00:31.521 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T09:00:31.521 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_49 2024-09-14T09:00:31.521 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:1 2024-09-14T09:00:31.521 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T09:00:31.521 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T09:00:31.521 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T09:00:31.521 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:1 2024-09-14T09:00:31.522 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T09:00:31.522 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T09:00:31.522 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T09:00:31.522 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=1 2024-09-14T09:00:31.522 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T09:00:31.522 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T09:00:31.522 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T09:00:31.522 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=1 2024-09-14T09:00:31.522 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T09:00:31.522 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T09:00:31.522 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:1 2024-09-14T09:00:31.522 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T09:00:31.522 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T09:00:31.522 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T09:00:31.522 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:1 2024-09-14T09:00:31.522 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T09:00:31.523 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T09:00:31.523 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T09:00:31.523 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T09:00:31.523 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T09:00:31.523 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T09:00:31.523 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T09:00:31.523 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=1 2024-09-14T09:00:31.523 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=1 2024-09-14T09:00:31.523 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T09:00:31.540 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T09:00:31.540 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T09:00:31.540 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T09:00:31.540 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_49 2024-09-14T09:00:31.622 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_49'] 2024-09-14T09:00:31.622 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T09:00:31.622 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T09:00:31.625 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T09:00:31.625 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_49 2024-09-14T09:00:31.625 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:2 2024-09-14T09:00:31.625 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T09:00:31.625 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T09:00:31.625 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-14T09:00:31.626 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:2 2024-09-14T09:00:31.626 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T09:00:31.626 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T09:00:31.626 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T09:00:31.626 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=2 2024-09-14T09:00:31.626 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T09:00:31.626 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T09:00:31.626 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T09:00:31.626 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=2 2024-09-14T09:00:31.626 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T09:00:31.626 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-14T09:00:31.626 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:2 2024-09-14T09:00:31.626 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T09:00:31.626 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T09:00:31.626 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T09:00:31.627 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:2 2024-09-14T09:00:31.627 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T09:00:31.627 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T09:00:31.627 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T09:00:31.627 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T09:00:31.627 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-14T09:00:31.627 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T09:00:31.627 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T09:00:31.627 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=2 2024-09-14T09:00:31.627 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=2 2024-09-14T09:00:31.627 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T09:00:31.643 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T09:00:31.643 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T09:00:31.643 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T09:00:31.643 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_49 2024-09-14T09:00:31.720 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_49'] 2024-09-14T09:00:31.720 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T09:00:31.720 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T09:00:31.723 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T09:00:31.723 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_49 2024-09-14T09:00:31.723 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:3 2024-09-14T09:00:31.723 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T09:00:31.723 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T09:00:31.723 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:3 cluster instance 2024-09-14T09:00:31.723 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:3 2024-09-14T09:00:31.723 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T09:00:31.724 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T09:00:31.724 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T09:00:31.724 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=3 2024-09-14T09:00:31.724 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-14T09:00:31.724 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T09:00:31.724 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T09:00:31.724 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=3 2024-09-14T09:00:31.724 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=3 2024-09-14T09:00:31.724 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:3 cluster1 2024-09-14T09:00:31.724 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:3 2024-09-14T09:00:31.724 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T09:00:31.724 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T09:00:31.724 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-14T09:00:31.724 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:3 2024-09-14T09:00:31.725 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T09:00:31.725 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T09:00:31.725 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T09:00:31.725 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=3 2024-09-14T09:00:31.725 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 3 = cluster1:3 ']' 2024-09-14T09:00:31.725 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T09:00:31.725 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T09:00:31.725 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=3 2024-09-14T09:00:31.725 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=3 2024-09-14T09:00:31.725 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-14T09:00:31.742 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T09:00:31.742 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T09:00:31.742 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T09:00:31.742 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_49 2024-09-14T09:00:31.822 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_49'] 2024-09-14T09:00:31.822 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T09:00:31.822 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T09:00:31.823 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T09:00:31.823 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sleep 4 2024-09-14T09:00:35.824 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T09:00:35.825 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ seq 0 3 2024-09-14T09:00:35.827 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T09:00:35.828 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_49 2024-09-14T09:00:35.828 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:0 2024-09-14T09:00:35.828 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T09:00:35.828 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T09:00:35.828 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T09:00:35.828 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:0 2024-09-14T09:00:35.828 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T09:00:35.828 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T09:00:35.828 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T09:00:35.828 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=0 2024-09-14T09:00:35.828 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T09:00:35.828 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T09:00:35.829 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T09:00:35.829 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=0 2024-09-14T09:00:35.829 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T09:00:35.829 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T09:00:35.829 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:0 2024-09-14T09:00:35.829 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T09:00:35.829 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T09:00:35.829 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T09:00:35.829 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:0 2024-09-14T09:00:35.829 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T09:00:35.829 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T09:00:35.829 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T09:00:35.829 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T09:00:35.830 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T09:00:35.830 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T09:00:35.830 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T09:00:35.830 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=0 2024-09-14T09:00:35.830 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=0 2024-09-14T09:00:35.830 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T09:00:35.849 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T09:00:35.850 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T09:00:35.850 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T09:00:35.850 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_49 2024-09-14T09:00:35.938 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_49'] 2024-09-14T09:00:35.938 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T09:00:35.938 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T09:00:35.941 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T09:00:35.941 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_49 2024-09-14T09:00:35.941 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:1 2024-09-14T09:00:35.941 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T09:00:35.941 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T09:00:35.941 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T09:00:35.941 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:1 2024-09-14T09:00:35.941 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T09:00:35.941 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T09:00:35.941 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T09:00:35.941 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=1 2024-09-14T09:00:35.941 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T09:00:35.942 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T09:00:35.942 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T09:00:35.942 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=1 2024-09-14T09:00:35.942 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T09:00:35.942 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T09:00:35.942 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:1 2024-09-14T09:00:35.942 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T09:00:35.942 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T09:00:35.942 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T09:00:35.942 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:1 2024-09-14T09:00:35.942 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T09:00:35.942 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T09:00:35.942 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T09:00:35.942 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T09:00:35.943 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T09:00:35.943 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T09:00:35.943 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T09:00:35.943 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=1 2024-09-14T09:00:35.943 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=1 2024-09-14T09:00:35.943 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T09:00:35.960 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T09:00:35.960 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T09:00:35.960 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T09:00:35.960 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_49 2024-09-14T09:00:36.042 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_49'] 2024-09-14T09:00:36.042 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T09:00:36.042 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T09:00:36.045 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T09:00:36.045 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_49 2024-09-14T09:00:36.045 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:2 2024-09-14T09:00:36.045 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T09:00:36.045 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T09:00:36.045 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-14T09:00:36.046 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:2 2024-09-14T09:00:36.046 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T09:00:36.046 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T09:00:36.046 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T09:00:36.046 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=2 2024-09-14T09:00:36.046 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T09:00:36.046 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T09:00:36.046 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T09:00:36.046 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=2 2024-09-14T09:00:36.046 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T09:00:36.046 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-14T09:00:36.046 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:2 2024-09-14T09:00:36.046 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T09:00:36.046 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T09:00:36.046 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T09:00:36.047 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:2 2024-09-14T09:00:36.047 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T09:00:36.047 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T09:00:36.047 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T09:00:36.047 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T09:00:36.047 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-14T09:00:36.047 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T09:00:36.047 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T09:00:36.047 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=2 2024-09-14T09:00:36.047 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=2 2024-09-14T09:00:36.047 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T09:00:36.064 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T09:00:36.064 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T09:00:36.064 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T09:00:36.064 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_49 2024-09-14T09:00:36.138 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_49'] 2024-09-14T09:00:36.139 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T09:00:36.139 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T09:00:36.142 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T09:00:36.142 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_49 2024-09-14T09:00:36.142 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:3 2024-09-14T09:00:36.142 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T09:00:36.142 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T09:00:36.142 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:3 cluster instance 2024-09-14T09:00:36.142 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:3 2024-09-14T09:00:36.142 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T09:00:36.142 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T09:00:36.142 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T09:00:36.142 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=3 2024-09-14T09:00:36.143 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-14T09:00:36.143 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T09:00:36.143 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T09:00:36.143 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=3 2024-09-14T09:00:36.143 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=3 2024-09-14T09:00:36.143 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:3 cluster1 2024-09-14T09:00:36.143 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:3 2024-09-14T09:00:36.143 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T09:00:36.143 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T09:00:36.143 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-14T09:00:36.143 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:3 2024-09-14T09:00:36.143 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T09:00:36.143 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T09:00:36.143 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T09:00:36.144 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=3 2024-09-14T09:00:36.144 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 3 = cluster1:3 ']' 2024-09-14T09:00:36.144 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T09:00:36.144 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T09:00:36.144 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=3 2024-09-14T09:00:36.144 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=3 2024-09-14T09:00:36.144 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-14T09:00:36.160 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T09:00:36.160 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T09:00:36.160 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T09:00:36.160 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_49 2024-09-14T09:00:36.240 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_49'] 2024-09-14T09:00:36.241 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T09:00:36.241 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T09:00:36.242 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T09:00:36.242 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sleep 8 2024-09-14T09:00:44.243 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T09:00:44.244 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ seq 0 3 2024-09-14T09:00:44.246 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T09:00:44.246 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_49 2024-09-14T09:00:44.246 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:0 2024-09-14T09:00:44.247 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T09:00:44.247 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T09:00:44.247 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T09:00:44.247 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:0 2024-09-14T09:00:44.247 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T09:00:44.247 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T09:00:44.247 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T09:00:44.247 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=0 2024-09-14T09:00:44.247 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T09:00:44.247 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T09:00:44.247 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T09:00:44.247 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=0 2024-09-14T09:00:44.248 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T09:00:44.248 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T09:00:44.248 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:0 2024-09-14T09:00:44.248 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T09:00:44.248 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T09:00:44.248 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T09:00:44.248 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:0 2024-09-14T09:00:44.248 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T09:00:44.248 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T09:00:44.248 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T09:00:44.248 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T09:00:44.248 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T09:00:44.248 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T09:00:44.248 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T09:00:44.248 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=0 2024-09-14T09:00:44.249 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=0 2024-09-14T09:00:44.249 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T09:00:44.269 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T09:00:44.269 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T09:00:44.269 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T09:00:44.269 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_49 2024-09-14T09:00:44.357 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_49'] 2024-09-14T09:00:44.357 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T09:00:44.357 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T09:00:44.360 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T09:00:44.361 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_49 2024-09-14T09:00:44.361 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:1 2024-09-14T09:00:44.361 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T09:00:44.361 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T09:00:44.361 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T09:00:44.361 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:1 2024-09-14T09:00:44.361 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T09:00:44.361 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T09:00:44.361 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T09:00:44.361 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=1 2024-09-14T09:00:44.361 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T09:00:44.361 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T09:00:44.361 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T09:00:44.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=1 2024-09-14T09:00:44.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T09:00:44.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T09:00:44.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:1 2024-09-14T09:00:44.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T09:00:44.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T09:00:44.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T09:00:44.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:1 2024-09-14T09:00:44.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T09:00:44.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T09:00:44.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T09:00:44.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T09:00:44.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T09:00:44.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T09:00:44.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T09:00:44.363 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=1 2024-09-14T09:00:44.363 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=1 2024-09-14T09:00:44.363 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T09:00:44.379 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T09:00:44.379 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T09:00:44.379 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T09:00:44.379 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_49 2024-09-14T09:00:44.461 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_49'] 2024-09-14T09:00:44.462 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T09:00:44.462 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T09:00:44.464 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T09:00:44.464 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_49 2024-09-14T09:00:44.464 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:2 2024-09-14T09:00:44.465 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T09:00:44.465 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T09:00:44.465 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-14T09:00:44.465 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:2 2024-09-14T09:00:44.465 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T09:00:44.465 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T09:00:44.465 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T09:00:44.465 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=2 2024-09-14T09:00:44.465 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T09:00:44.465 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T09:00:44.465 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T09:00:44.465 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=2 2024-09-14T09:00:44.465 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T09:00:44.465 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-14T09:00:44.466 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:2 2024-09-14T09:00:44.466 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T09:00:44.466 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T09:00:44.466 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T09:00:44.466 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:2 2024-09-14T09:00:44.466 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T09:00:44.466 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T09:00:44.466 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T09:00:44.466 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T09:00:44.466 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-14T09:00:44.466 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T09:00:44.466 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T09:00:44.466 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=2 2024-09-14T09:00:44.466 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=2 2024-09-14T09:00:44.467 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T09:00:44.483 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T09:00:44.483 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T09:00:44.483 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T09:00:44.483 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_49 2024-09-14T09:00:44.558 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_49'] 2024-09-14T09:00:44.558 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T09:00:44.558 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T09:00:44.561 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T09:00:44.561 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_49 2024-09-14T09:00:44.561 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:3 2024-09-14T09:00:44.561 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T09:00:44.561 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T09:00:44.561 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:3 cluster instance 2024-09-14T09:00:44.561 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:3 2024-09-14T09:00:44.561 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T09:00:44.561 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T09:00:44.561 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T09:00:44.561 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=3 2024-09-14T09:00:44.561 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-14T09:00:44.562 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T09:00:44.562 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T09:00:44.562 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=3 2024-09-14T09:00:44.562 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=3 2024-09-14T09:00:44.562 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:3 cluster1 2024-09-14T09:00:44.562 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:3 2024-09-14T09:00:44.562 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T09:00:44.562 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T09:00:44.562 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-14T09:00:44.562 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:3 2024-09-14T09:00:44.562 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T09:00:44.562 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T09:00:44.562 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T09:00:44.562 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=3 2024-09-14T09:00:44.562 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 3 = cluster1:3 ']' 2024-09-14T09:00:44.563 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T09:00:44.563 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T09:00:44.563 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=3 2024-09-14T09:00:44.563 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=3 2024-09-14T09:00:44.563 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-14T09:00:44.580 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T09:00:44.581 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T09:00:44.581 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T09:00:44.581 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_49 2024-09-14T09:00:44.662 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_49'] 2024-09-14T09:00:44.662 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T09:00:44.663 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T09:00:44.663 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T09:00:44.663 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sleep 8 2024-09-14T09:00:52.665 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T09:00:52.665 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ seq 0 3 2024-09-14T09:00:52.668 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T09:00:52.668 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_49 2024-09-14T09:00:52.668 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:0 2024-09-14T09:00:52.668 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T09:00:52.669 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T09:00:52.669 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T09:00:52.669 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:0 2024-09-14T09:00:52.669 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T09:00:52.669 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T09:00:52.669 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T09:00:52.669 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=0 2024-09-14T09:00:52.669 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T09:00:52.669 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T09:00:52.669 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T09:00:52.669 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=0 2024-09-14T09:00:52.669 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T09:00:52.669 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T09:00:52.669 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:0 2024-09-14T09:00:52.670 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T09:00:52.670 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T09:00:52.670 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T09:00:52.670 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:0 2024-09-14T09:00:52.670 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T09:00:52.670 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T09:00:52.670 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T09:00:52.670 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T09:00:52.670 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T09:00:52.670 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T09:00:52.670 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T09:00:52.670 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=0 2024-09-14T09:00:52.670 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=0 2024-09-14T09:00:52.670 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T09:00:52.689 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T09:00:52.689 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T09:00:52.689 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T09:00:52.690 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_49 2024-09-14T09:00:52.778 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_49'] 2024-09-14T09:00:52.778 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T09:00:52.778 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T09:00:52.782 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T09:00:52.782 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_49 2024-09-14T09:00:52.782 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:1 2024-09-14T09:00:52.782 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T09:00:52.782 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T09:00:52.782 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T09:00:52.782 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:1 2024-09-14T09:00:52.782 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T09:00:52.782 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T09:00:52.782 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T09:00:52.782 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=1 2024-09-14T09:00:52.782 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T09:00:52.783 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T09:00:52.783 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T09:00:52.783 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=1 2024-09-14T09:00:52.783 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T09:00:52.783 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T09:00:52.783 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:1 2024-09-14T09:00:52.783 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T09:00:52.783 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T09:00:52.783 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T09:00:52.783 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:1 2024-09-14T09:00:52.783 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T09:00:52.783 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T09:00:52.783 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T09:00:52.783 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T09:00:52.783 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T09:00:52.784 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T09:00:52.784 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T09:00:52.784 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=1 2024-09-14T09:00:52.784 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=1 2024-09-14T09:00:52.784 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T09:00:52.801 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T09:00:52.801 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T09:00:52.801 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T09:00:52.801 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_49 2024-09-14T09:00:52.882 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_49'] 2024-09-14T09:00:52.883 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T09:00:52.883 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T09:00:52.886 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T09:00:52.886 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_49 2024-09-14T09:00:52.886 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:2 2024-09-14T09:00:52.886 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T09:00:52.886 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T09:00:52.886 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-14T09:00:52.886 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:2 2024-09-14T09:00:52.886 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T09:00:52.886 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T09:00:52.886 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T09:00:52.887 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=2 2024-09-14T09:00:52.887 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T09:00:52.887 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T09:00:52.887 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T09:00:52.887 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=2 2024-09-14T09:00:52.887 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T09:00:52.887 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-14T09:00:52.887 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:2 2024-09-14T09:00:52.887 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T09:00:52.887 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T09:00:52.887 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T09:00:52.887 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:2 2024-09-14T09:00:52.887 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T09:00:52.887 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T09:00:52.887 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T09:00:52.888 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T09:00:52.888 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-14T09:00:52.888 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T09:00:52.888 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T09:00:52.888 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=2 2024-09-14T09:00:52.888 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=2 2024-09-14T09:00:52.888 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T09:00:52.904 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T09:00:52.904 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T09:00:52.904 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T09:00:52.904 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_49 2024-09-14T09:00:52.979 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_49'] 2024-09-14T09:00:52.979 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T09:00:52.979 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T09:00:52.982 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T09:00:52.983 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_49 2024-09-14T09:00:52.983 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:3 2024-09-14T09:00:52.983 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T09:00:52.983 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T09:00:52.983 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:3 cluster instance 2024-09-14T09:00:52.983 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:3 2024-09-14T09:00:52.983 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T09:00:52.983 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T09:00:52.983 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T09:00:52.983 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=3 2024-09-14T09:00:52.983 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-14T09:00:52.983 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T09:00:52.983 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T09:00:52.984 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=3 2024-09-14T09:00:52.984 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=3 2024-09-14T09:00:52.984 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:3 cluster1 2024-09-14T09:00:52.984 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:3 2024-09-14T09:00:52.984 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T09:00:52.984 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T09:00:52.984 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-14T09:00:52.984 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:3 2024-09-14T09:00:52.984 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T09:00:52.984 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T09:00:52.984 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T09:00:52.984 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=3 2024-09-14T09:00:52.984 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 3 = cluster1:3 ']' 2024-09-14T09:00:52.984 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T09:00:52.984 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T09:00:52.985 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=3 2024-09-14T09:00:52.985 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=3 2024-09-14T09:00:52.985 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-14T09:00:53.001 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T09:00:53.001 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T09:00:53.001 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T09:00:53.001 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_49 2024-09-14T09:00:53.080 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_49'] 2024-09-14T09:00:53.081 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T09:00:53.081 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T09:00:53.082 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T09:00:53.082 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sleep 8 2024-09-14T09:01:01.083 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T09:01:01.083 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ seq 0 3 2024-09-14T09:01:01.086 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T09:01:01.086 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_49 2024-09-14T09:01:01.086 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:0 2024-09-14T09:01:01.086 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T09:01:01.087 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T09:01:01.087 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T09:01:01.087 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:0 2024-09-14T09:01:01.087 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T09:01:01.087 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T09:01:01.087 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T09:01:01.087 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=0 2024-09-14T09:01:01.087 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T09:01:01.087 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T09:01:01.087 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T09:01:01.087 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=0 2024-09-14T09:01:01.087 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T09:01:01.087 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T09:01:01.087 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:0 2024-09-14T09:01:01.087 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T09:01:01.088 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T09:01:01.088 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T09:01:01.088 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:0 2024-09-14T09:01:01.088 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T09:01:01.088 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T09:01:01.088 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T09:01:01.088 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T09:01:01.088 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T09:01:01.088 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T09:01:01.088 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T09:01:01.088 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=0 2024-09-14T09:01:01.088 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=0 2024-09-14T09:01:01.088 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T09:01:01.110 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T09:01:01.110 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T09:01:01.110 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T09:01:01.110 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_49 2024-09-14T09:01:01.199 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_49'] 2024-09-14T09:01:01.199 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T09:01:01.199 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T09:01:01.204 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T09:01:01.204 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_49 2024-09-14T09:01:01.205 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:1 2024-09-14T09:01:01.205 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T09:01:01.205 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T09:01:01.205 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T09:01:01.205 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:1 2024-09-14T09:01:01.205 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T09:01:01.205 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T09:01:01.205 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T09:01:01.205 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=1 2024-09-14T09:01:01.206 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T09:01:01.206 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T09:01:01.206 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T09:01:01.206 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=1 2024-09-14T09:01:01.206 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T09:01:01.206 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T09:01:01.206 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:1 2024-09-14T09:01:01.206 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T09:01:01.206 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T09:01:01.206 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T09:01:01.206 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:1 2024-09-14T09:01:01.206 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T09:01:01.206 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T09:01:01.206 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T09:01:01.206 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T09:01:01.206 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T09:01:01.207 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T09:01:01.207 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T09:01:01.207 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=1 2024-09-14T09:01:01.207 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=1 2024-09-14T09:01:01.207 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T09:01:01.221 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T09:01:01.222 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T09:01:01.222 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T09:01:01.222 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_49 2024-09-14T09:01:01.307 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_49'] 2024-09-14T09:01:01.307 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T09:01:01.307 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T09:01:01.310 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T09:01:01.310 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_49 2024-09-14T09:01:01.310 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:2 2024-09-14T09:01:01.310 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T09:01:01.311 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T09:01:01.311 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-14T09:01:01.311 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:2 2024-09-14T09:01:01.311 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T09:01:01.311 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T09:01:01.311 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T09:01:01.311 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=2 2024-09-14T09:01:01.311 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T09:01:01.311 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T09:01:01.311 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T09:01:01.311 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=2 2024-09-14T09:01:01.312 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T09:01:01.312 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-14T09:01:01.312 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:2 2024-09-14T09:01:01.312 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T09:01:01.312 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T09:01:01.312 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T09:01:01.312 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:2 2024-09-14T09:01:01.312 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T09:01:01.312 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T09:01:01.312 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T09:01:01.312 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T09:01:01.312 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-14T09:01:01.312 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T09:01:01.313 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T09:01:01.313 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=2 2024-09-14T09:01:01.313 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=2 2024-09-14T09:01:01.313 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T09:01:01.329 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T09:01:01.329 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T09:01:01.329 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T09:01:01.329 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_49 2024-09-14T09:01:01.421 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ return 0 2024-09-14T09:01:01.422 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T09:01:01.422 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ echo ' "state": "Replaying"' 2024-09-14T09:01:01.422 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep -i Replaying 2024-09-14T09:01:01.423 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: "state": "Replaying" 2024-09-14T09:01:01.427 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_state=started 2024-09-14T09:01:01.427 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test started = started 2024-09-14T09:01:01.427 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:01:01.428 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_49 2024-09-14T09:01:01.428 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T09:01:01.428 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:01:01.428 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:01:01.428 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_49 2024-09-14T09:01:01.428 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = journal ']' 2024-09-14T09:01:01.428 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = snapshot ']' 2024-09-14T09:01:01.428 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_49 2024-09-14T09:01:01.429 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T09:01:01.429 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:01:01.429 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:01:01.429 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_49 2024-09-14T09:01:01.429 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster2-mirror-image_49.status 2024-09-14T09:01:01.429 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster2-mirror-image_49.status 2024-09-14T09:01:01.429 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T09:01:01.429 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_49.status 2024-09-14T09:01:01.430 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster1-mirror-image_49.status 2024-09-14T09:01:01.430 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster1-mirror-image_49.status 2024-09-14T09:01:01.430 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T09:01:01.430 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_status_log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_49.status 2024-09-14T09:01:01.430 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ mirror_image_snapshot cluster2 mirror image_49 2024-09-14T09:01:01.430 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:01:01.430 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:01:01.430 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_49 2024-09-14T09:01:01.430 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_49 2024-09-14T09:01:01.628 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Snapshot ID: 182 2024-09-14T09:01:01.635 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_49 /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_49.status 2024-09-14T09:01:01.635 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:01:01.635 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:01:01.635 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_49 2024-09-14T09:01:01.635 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_49.status 2024-09-14T09:01:01.635 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_49 --format xml 2024-09-14T09:01:01.635 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T09:01:01.885 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-14T09:01:01.885 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snapshot_id=182 2024-09-14T09:01:01.886 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T09:01:01.886 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T09:01:01.886 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.2 2024-09-14T09:01:02.089 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_49 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_49.status 2024-09-14T09:01:02.089 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:01:02.089 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:01:02.089 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_49 2024-09-14T09:01:02.089 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_49.status 2024-09-14T09:01:02.089 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_49 --format xml 2024-09-14T09:01:02.090 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T09:01:02.137 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T09:01:02.139 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=83 2024-09-14T09:01:02.139 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 182 = 83 2024-09-14T09:01:02.139 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T09:01:02.139 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.4 2024-09-14T09:01:02.540 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_49 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_49.status 2024-09-14T09:01:02.541 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:01:02.541 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:01:02.541 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_49 2024-09-14T09:01:02.541 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_49.status 2024-09-14T09:01:02.541 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_49 --format xml 2024-09-14T09:01:02.541 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T09:01:02.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T09:01:02.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T09:01:02.591 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T09:01:02.591 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T09:01:02.591 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T09:01:02.591 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T09:01:02.591 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 182 = '' 2024-09-14T09:01:02.592 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T09:01:02.592 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.8 2024-09-14T09:01:03.393 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_49 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_49.status 2024-09-14T09:01:03.393 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:01:03.393 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:01:03.394 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_49 2024-09-14T09:01:03.394 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_49.status 2024-09-14T09:01:03.394 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_49 --format xml 2024-09-14T09:01:03.394 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T09:01:03.448 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T09:01:03.448 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T09:01:03.449 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T09:01:03.449 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T09:01:03.449 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T09:01:03.450 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T09:01:03.450 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 182 = '' 2024-09-14T09:01:03.450 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T09:01:03.450 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1.6 2024-09-14T09:01:05.052 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_49 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_49.status 2024-09-14T09:01:05.052 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:01:05.052 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:01:05.052 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_49 2024-09-14T09:01:05.052 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_49.status 2024-09-14T09:01:05.052 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_49 --format xml 2024-09-14T09:01:05.052 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T09:01:05.109 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T09:01:05.111 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=182 2024-09-14T09:01:05.111 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 182 = 182 2024-09-14T09:01:05.111 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:01:05.111 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snap_present cluster1 mirror image_49 snap 2024-09-14T09:01:05.111 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:01:05.111 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:01:05.112 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_49 2024-09-14T09:01:05.112 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T09:01:05.112 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T09:01:05.112 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T09:01:05.112 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T09:01:06.112 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 info mirror/image_49@snap 2024-09-14T09:01:06.164 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_49': 2024-09-14T09:01:06.164 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T09:01:06.164 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T09:01:06.164 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 2 2024-09-14T09:01:06.164 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 10bdb7cc2bc3 2024-09-14T09:01:06.164 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.10bdb7cc2bc3 2024-09-14T09:01:06.164 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T09:01:06.165 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering, non-primary 2024-09-14T09:01:06.165 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T09:01:06.165 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T09:01:06.165 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:53:12 2024 2024-09-14T09:01:06.165 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:53:12 2024 2024-09-14T09:01:06.165 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:53:12 2024 2024-09-14T09:01:06.165 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: protected: False 2024-09-14T09:01:06.165 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T09:01:06.165 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T09:01:06.165 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: e852057c-0945-4c55-b535-c147c24f9585 2024-09-14T09:01:06.165 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: false 2024-09-14T09:01:06.168 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:01:06.168 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ compare_image_snaps mirror image_49 snap 2024-09-14T09:01:06.168 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:01:06.168 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_49 2024-09-14T09:01:06.168 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T09:01:06.168 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local ret=0 2024-09-14T09:01:06.168 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local rmt_export=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_49.export 2024-09-14T09:01:06.169 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local loc_export=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_49.export 2024-09-14T09:01:06.169 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_49.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_49.export 2024-09-14T09:01:06.169 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror export image_49@snap /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_49.export 2024-09-14T09:01:06.380 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T09:01:06.387 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 -p mirror export image_49@snap /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_49.export 2024-09-14T09:01:06.651 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T09:01:06.658 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cmp /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_49.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_49.export 2024-09-14T09:01:06.714 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_49.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_49.export 2024-09-14T09:01:06.747 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:01:06.747 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:01:06.747 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_50 2024-09-14T09:01:06.747 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_snap cluster2 mirror image_50 snap 2024-09-14T09:01:06.747 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:01:06.747 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:01:06.747 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_50 2024-09-14T09:01:06.747 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T09:01:06.747 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror snap create image_50@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T09:01:07.632 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_started cluster1 mirror image_50 2024-09-14T09:01:07.633 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:01:07.633 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:01:07.633 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_50 2024-09-14T09:01:07.633 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_replay_state cluster1 mirror image_50 started 2024-09-14T09:01:07.633 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:01:07.633 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:01:07.633 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_50 2024-09-14T09:01:07.633 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=started 2024-09-14T09:01:07.633 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T09:01:07.633 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T09:01:07.633 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T09:01:08.633 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_replay_state cluster1 mirror image_50 started 2024-09-14T09:01:08.633 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:01:08.634 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:01:08.634 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_50 2024-09-14T09:01:08.634 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=started 2024-09-14T09:01:08.634 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_result 2024-09-14T09:01:08.634 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=stopped 2024-09-14T09:01:08.635 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_50 2024-09-14T09:01:08.635 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_instance=cluster1 2024-09-14T09:01:08.635 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T09:01:08.635 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:01:08.635 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ grep -i state 2024-09-14T09:01:08.635 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance=cluster1 2024-09-14T09:01:08.635 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local loop_instance 2024-09-14T09:01:08.635 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T09:01:08.635 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sleep 0 2024-09-14T09:01:08.636 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T09:01:08.638 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ seq 0 3 2024-09-14T09:01:08.638 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T09:01:08.638 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_50 2024-09-14T09:01:08.639 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:0 2024-09-14T09:01:08.639 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T09:01:08.639 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T09:01:08.639 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T09:01:08.639 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:0 2024-09-14T09:01:08.639 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T09:01:08.639 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T09:01:08.639 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T09:01:08.639 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=0 2024-09-14T09:01:08.639 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T09:01:08.639 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T09:01:08.639 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T09:01:08.639 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=0 2024-09-14T09:01:08.640 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T09:01:08.640 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T09:01:08.640 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:0 2024-09-14T09:01:08.640 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T09:01:08.640 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T09:01:08.640 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T09:01:08.640 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:0 2024-09-14T09:01:08.640 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T09:01:08.640 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T09:01:08.640 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T09:01:08.640 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=0 2024-09-14T09:01:08.640 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T09:01:08.640 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T09:01:08.640 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T09:01:08.641 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=0 2024-09-14T09:01:08.641 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=0 2024-09-14T09:01:08.641 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T09:01:08.660 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T09:01:08.660 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T09:01:08.660 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T09:01:08.660 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_50 2024-09-14T09:01:08.749 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_50'] 2024-09-14T09:01:08.749 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T09:01:08.749 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T09:01:08.752 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T09:01:08.752 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_50 2024-09-14T09:01:08.752 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:1 2024-09-14T09:01:08.752 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T09:01:08.752 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T09:01:08.752 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T09:01:08.752 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:1 2024-09-14T09:01:08.753 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T09:01:08.753 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T09:01:08.753 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T09:01:08.753 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=1 2024-09-14T09:01:08.753 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T09:01:08.753 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T09:01:08.753 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T09:01:08.753 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=1 2024-09-14T09:01:08.753 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T09:01:08.753 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T09:01:08.753 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:1 2024-09-14T09:01:08.753 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T09:01:08.753 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T09:01:08.753 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T09:01:08.753 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:1 2024-09-14T09:01:08.754 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T09:01:08.754 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T09:01:08.754 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T09:01:08.754 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=1 2024-09-14T09:01:08.754 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T09:01:08.754 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T09:01:08.754 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T09:01:08.754 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=1 2024-09-14T09:01:08.754 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=1 2024-09-14T09:01:08.754 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T09:01:08.773 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T09:01:08.773 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T09:01:08.774 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T09:01:08.774 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_50 2024-09-14T09:01:08.854 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_50'] 2024-09-14T09:01:08.854 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T09:01:08.854 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T09:01:08.857 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T09:01:08.857 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_50 2024-09-14T09:01:08.857 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:2 2024-09-14T09:01:08.858 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T09:01:08.858 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T09:01:08.858 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-14T09:01:08.858 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:2 2024-09-14T09:01:08.858 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T09:01:08.858 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T09:01:08.858 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T09:01:08.858 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=2 2024-09-14T09:01:08.858 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T09:01:08.858 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T09:01:08.858 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T09:01:08.858 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=2 2024-09-14T09:01:08.858 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T09:01:08.859 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-14T09:01:08.859 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:2 2024-09-14T09:01:08.859 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T09:01:08.859 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T09:01:08.859 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T09:01:08.859 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:2 2024-09-14T09:01:08.859 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T09:01:08.859 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T09:01:08.859 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T09:01:08.859 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=2 2024-09-14T09:01:08.860 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-14T09:01:08.860 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T09:01:08.860 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T09:01:08.860 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=2 2024-09-14T09:01:08.860 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=2 2024-09-14T09:01:08.860 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T09:01:08.875 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T09:01:08.876 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T09:01:08.876 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T09:01:08.876 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_50 2024-09-14T09:01:08.958 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:Invalid command: unused arguments: ['mirror/image_50'] 2024-09-14T09:01:08.958 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T09:01:08.958 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:admin_socket: invalid command 2024-09-14T09:01:08.960 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T09:01:08.961 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_50 2024-09-14T09:01:08.961 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1:3 2024-09-14T09:01:08.961 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ shift 2024-09-14T09:01:08.961 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance 2024-09-14T09:01:08.961 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ set_cluster_instance cluster1:3 cluster instance 2024-09-14T09:01:08.961 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local val=cluster1:3 2024-09-14T09:01:08.961 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster_var_name=cluster 2024-09-14T09:01:08.961 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local instance_var_name=instance 2024-09-14T09:01:08.961 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T09:01:08.961 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=3 2024-09-14T09:01:08.961 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-14T09:01:08.961 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval cluster=cluster1 2024-09-14T09:01:08.961 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T09:01:08.961 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ eval instance=3 2024-09-14T09:01:08.962 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=3 2024-09-14T09:01:08.962 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ daemon_asok_file cluster1:3 cluster1 2024-09-14T09:01:08.962 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local local_cluster=cluster1:3 2024-09-14T09:01:08.962 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster=cluster1 2024-09-14T09:01:08.962 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance 2024-09-14T09:01:08.962 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-14T09:01:08.962 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local val=cluster1:3 2024-09-14T09:01:08.962 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T09:01:08.962 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ local instance_var_name=instance 2024-09-14T09:01:08.962 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ cluster=cluster1 2024-09-14T09:01:08.962 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ instance=3 2024-09-14T09:01:08.963 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ '[' 3 = cluster1:3 ']' 2024-09-14T09:01:08.963 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval local_cluster=cluster1 2024-09-14T09:01:08.963 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ local_cluster=cluster1 2024-09-14T09:01:08.963 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ eval instance=3 2024-09-14T09:01:08.963 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ instance=3 2024-09-14T09:01:08.963 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-14T09:01:08.980 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T09:01:08.981 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T09:01:08.981 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T09:01:08.981 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_50 2024-09-14T09:01:09.073 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ return 0 2024-09-14T09:01:09.074 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T09:01:09.074 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ grep -i Replaying 2024-09-14T09:01:09.074 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ echo ' "state": "Replaying"' 2024-09-14T09:01:09.075 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: "state": "Replaying" 2024-09-14T09:01:09.077 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_state=started 2024-09-14T09:01:09.077 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test started = started 2024-09-14T09:01:09.077 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:01:09.077 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_50 2024-09-14T09:01:09.077 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T09:01:09.078 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:01:09.078 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:01:09.078 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_50 2024-09-14T09:01:09.078 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = journal ']' 2024-09-14T09:01:09.078 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' snapshot = snapshot ']' 2024-09-14T09:01:09.078 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_50 2024-09-14T09:01:09.078 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_cluster=cluster1 2024-09-14T09:01:09.078 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:01:09.078 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:01:09.078 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_50 2024-09-14T09:01:09.078 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster2-mirror-image_50.status 2024-09-14T09:01:09.078 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster2-mirror-image_50.status 2024-09-14T09:01:09.078 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T09:01:09.079 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local status_log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_50.status 2024-09-14T09:01:09.079 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ mkfname cluster1-mirror-image_50.status 2024-09-14T09:01:09.079 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ echo cluster1-mirror-image_50.status 2024-09-14T09:01:09.079 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-14T09:01:09.080 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local local_status_log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_50.status 2024-09-14T09:01:09.080 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ mirror_image_snapshot cluster2 mirror image_50 2024-09-14T09:01:09.080 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:01:09.080 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:01:09.080 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_50 2024-09-14T09:01:09.080 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_50 2024-09-14T09:01:09.646 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Snapshot ID: 184 2024-09-14T09:01:09.653 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_50 /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_50.status 2024-09-14T09:01:09.653 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:01:09.653 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:01:09.653 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_50 2024-09-14T09:01:09.653 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_50.status 2024-09-14T09:01:09.653 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_50 --format xml 2024-09-14T09:01:09.653 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T09:01:09.699 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-14T09:01:09.701 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snapshot_id=184 2024-09-14T09:01:09.701 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T09:01:09.701 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T09:01:09.701 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.2 2024-09-14T09:01:09.902 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_50 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_50.status 2024-09-14T09:01:09.902 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:01:09.902 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:01:09.903 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_50 2024-09-14T09:01:09.903 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_50.status 2024-09-14T09:01:09.903 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_50 --format xml 2024-09-14T09:01:09.903 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T09:01:09.952 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T09:01:09.954 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=84 2024-09-14T09:01:09.954 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 184 = 84 2024-09-14T09:01:09.954 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T09:01:09.954 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.4 2024-09-14T09:01:10.356 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_50 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_50.status 2024-09-14T09:01:10.356 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:01:10.356 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:01:10.356 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_50 2024-09-14T09:01:10.356 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_50.status 2024-09-14T09:01:10.356 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_50 --format xml 2024-09-14T09:01:10.356 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T09:01:10.409 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T09:01:10.411 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=84 2024-09-14T09:01:10.411 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 184 = 84 2024-09-14T09:01:10.411 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T09:01:10.411 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.8 2024-09-14T09:01:11.212 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_50 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_50.status 2024-09-14T09:01:11.212 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:01:11.213 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:01:11.213 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_50 2024-09-14T09:01:11.213 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_50.status 2024-09-14T09:01:11.213 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_50 --format xml 2024-09-14T09:01:11.213 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T09:01:11.268 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T09:01:11.268 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T09:01:11.270 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:-:1.1: Document is empty 2024-09-14T09:01:11.270 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: 2024-09-14T09:01:11.270 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:^ 2024-09-14T09:01:11.270 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id= 2024-09-14T09:01:11.270 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 184 = '' 2024-09-14T09:01:11.270 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T09:01:11.270 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1.6 2024-09-14T09:01:12.871 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_50 /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_50.status 2024-09-14T09:01:12.872 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:01:12.872 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:01:12.872 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_50 2024-09-14T09:01:12.872 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local log=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_50.status 2024-09-14T09:01:12.872 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_50 --format xml 2024-09-14T09:01:12.872 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-14T09:01:12.925 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-14T09:01:12.927 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local primary_snapshot_id=184 2024-09-14T09:01:12.927 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 184 = 184 2024-09-14T09:01:12.927 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:01:12.927 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_snap_present cluster1 mirror image_50 snap 2024-09-14T09:01:12.927 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:01:12.927 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:01:12.927 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_50 2024-09-14T09:01:12.928 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T09:01:12.928 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T09:01:12.928 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T09:01:12.928 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T09:01:13.928 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 info mirror/image_50@snap 2024-09-14T09:01:13.975 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_50': 2024-09-14T09:01:13.975 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 128 MiB in 32 objects 2024-09-14T09:01:13.975 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T09:01:13.975 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 2 2024-09-14T09:01:13.975 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 11892faa76af 2024-09-14T09:01:13.975 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.11892faa76af 2024-09-14T09:01:13.976 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T09:01:13.976 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering, non-primary 2024-09-14T09:01:13.976 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T09:01:13.976 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T09:01:13.976 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 08:53:18 2024 2024-09-14T09:01:13.976 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 08:53:18 2024 2024-09-14T09:01:13.976 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 08:53:18 2024 2024-09-14T09:01:13.976 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: protected: False 2024-09-14T09:01:13.976 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T09:01:13.976 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T09:01:13.976 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: dc8ae139-d4cc-4aab-82f3-d300cef64dfd 2024-09-14T09:01:13.976 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: false 2024-09-14T09:01:13.979 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:01:13.979 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ compare_image_snaps mirror image_50 snap 2024-09-14T09:01:13.979 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:01:13.979 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_50 2024-09-14T09:01:13.979 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap_name=snap 2024-09-14T09:01:13.980 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local ret=0 2024-09-14T09:01:13.980 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local rmt_export=/tmp/tmp.IUd9LESfOA/cluster2-mirror-image_50.export 2024-09-14T09:01:13.980 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local loc_export=/tmp/tmp.IUd9LESfOA/cluster1-mirror-image_50.export 2024-09-14T09:01:13.980 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_50.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_50.export 2024-09-14T09:01:13.980 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 -p mirror export image_50@snap /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_50.export 2024-09-14T09:01:14.175 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T09:01:14.184 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 -p mirror export image_50@snap /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_50.export 2024-09-14T09:01:14.441 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T09:01:14.448 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cmp /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_50.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_50.export 2024-09-14T09:01:14.502 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -f /tmp/tmp.IUd9LESfOA/cluster2-mirror-image_50.export /tmp/tmp.IUd9LESfOA/cluster1-mirror-image_50.export 2024-09-14T09:01:14.534 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:01:14.534 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ testlog 'TEST: delete many images' 2024-09-14T09:01:14.534 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ tee -a /tmp/tmp.IUd9LESfOA/rbd-mirror.test.log 2024-09-14T09:01:14.535 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ date '+%F %T' 2024-09-14T09:01:14.535 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ echo 2024-09-14 09:01:14 TEST: delete many images 2024-09-14T09:01:14.535 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:2024-09-14 09:01:14 TEST: delete many images 2024-09-14T09:01:14.536 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ seq 1 50 2024-09-14T09:01:14.537 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:01:14.537 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_1 2024-09-14T09:01:14.537 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_snapshot cluster2 mirror image_1 snap 2024-09-14T09:01:14.537 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:01:14.537 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:01:14.537 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_1 2024-09-14T09:01:14.537 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap=snap 2024-09-14T09:01:14.537 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap rm mirror/image_1@snap 2024-09-14T09:01:14.639 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing snap: 100% complete...done. 2024-09-14T09:01:14.652 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image_retry cluster2 mirror image_1 2024-09-14T09:01:14.652 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:01:14.652 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:01:14.653 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_1 2024-09-14T09:01:14.653 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:01:14.653 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0 2024-09-14T09:01:14.653 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_1 2024-09-14T09:01:14.654 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:01:14.654 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:01:14.654 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_1 2024-09-14T09:01:14.654 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_1 2024-09-14T09:01:14.703 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_1 2024-09-14T09:01:14.806 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi164.stderr:2024-09-14T09:01:14.806+0000 7f7e31d23640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55b570dbe9c0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-14T09:01:14.807 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi164.stderr:2024-09-14T09:01:14.806+0000 7f7e38d31640 -1 rbd::mirror::ImageReplayer: 0x55b5720e2c80 [3/06bb8003-76f2-47e8-a72f-feadf752d8d3] operator(): start failed: (85) Interrupted system call should be restarted 2024-09-14T09:01:15.773 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T09:01:15.777 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:01:15.777 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:01:15.777 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_2 2024-09-14T09:01:15.777 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_snapshot cluster2 mirror image_2 snap 2024-09-14T09:01:15.777 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:01:15.777 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:01:15.777 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_2 2024-09-14T09:01:15.777 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap=snap 2024-09-14T09:01:15.777 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap rm mirror/image_2@snap 2024-09-14T09:01:16.660 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing snap: 100% complete...done. 2024-09-14T09:01:16.671 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image_retry cluster2 mirror image_2 2024-09-14T09:01:16.671 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:01:16.671 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:01:16.671 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_2 2024-09-14T09:01:16.672 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:01:16.672 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0 2024-09-14T09:01:16.673 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_2 2024-09-14T09:01:16.673 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:01:16.673 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:01:16.673 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_2 2024-09-14T09:01:16.673 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_2 2024-09-14T09:01:16.727 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_2 2024-09-14T09:01:16.823 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi164.stderr:2024-09-14T09:01:16.823+0000 7f3753e08640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55c94c51e820 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-14T09:01:16.823 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi164.stderr:2024-09-14T09:01:16.823+0000 7f375a615640 -1 rbd::mirror::ImageReplayer: 0x55c94bf17180 [3/f36140bd-57e9-420f-8e29-5672a8b58d3d] operator(): start failed: (85) Interrupted system call should be restarted 2024-09-14T09:01:17.779 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T09:01:17.783 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:01:17.783 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:01:17.783 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_3 2024-09-14T09:01:17.783 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_snapshot cluster2 mirror image_3 snap 2024-09-14T09:01:17.783 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:01:17.783 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:01:17.783 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_3 2024-09-14T09:01:17.783 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap=snap 2024-09-14T09:01:17.783 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap rm mirror/image_3@snap 2024-09-14T09:01:18.655 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing snap: 100% complete...done. 2024-09-14T09:01:18.666 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image_retry cluster2 mirror image_3 2024-09-14T09:01:18.666 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:01:18.666 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:01:18.666 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_3 2024-09-14T09:01:18.666 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:01:18.666 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0 2024-09-14T09:01:18.667 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_3 2024-09-14T09:01:18.667 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:01:18.667 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:01:18.667 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_3 2024-09-14T09:01:18.667 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_3 2024-09-14T09:01:18.717 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_3 2024-09-14T09:01:18.812 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi164.stderr:2024-09-14T09:01:18.812+0000 7f3753e08640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55c94c51f860 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-14T09:01:18.813 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi164.stderr:2024-09-14T09:01:18.812+0000 7f375a615640 -1 rbd::mirror::ImageReplayer: 0x55c94bf16a00 [3/c4fe1bd8-3565-46e8-87a1-574d1ec9a443] operator(): start failed: (85) Interrupted system call should be restarted 2024-09-14T09:01:19.778 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T09:01:19.782 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:01:19.782 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:01:19.782 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_4 2024-09-14T09:01:19.782 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_snapshot cluster2 mirror image_4 snap 2024-09-14T09:01:19.782 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:01:19.782 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:01:19.782 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_4 2024-09-14T09:01:19.782 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap=snap 2024-09-14T09:01:19.782 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap rm mirror/image_4@snap 2024-09-14T09:01:20.664 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing snap: 100% complete...done. 2024-09-14T09:01:20.674 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image_retry cluster2 mirror image_4 2024-09-14T09:01:20.674 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:01:20.674 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:01:20.674 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_4 2024-09-14T09:01:20.674 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:01:20.674 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0 2024-09-14T09:01:20.675 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_4 2024-09-14T09:01:20.675 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:01:20.675 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:01:20.675 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_4 2024-09-14T09:01:20.675 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_4 2024-09-14T09:01:20.732 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_4 2024-09-14T09:01:20.827 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi164.stderr:2024-09-14T09:01:20.826+0000 7f7e32524640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55b570dbf1e0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-14T09:01:20.827 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi164.stderr:2024-09-14T09:01:20.826+0000 7f7e38d31640 -1 rbd::mirror::ImageReplayer: 0x55b5720e3900 [3/3fcd7b58-faf2-4823-8a72-3bd4c17d6db0] operator(): start failed: (85) Interrupted system call should be restarted 2024-09-14T09:01:21.789 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T09:01:21.793 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:01:21.793 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:01:21.793 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_5 2024-09-14T09:01:21.793 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_snapshot cluster2 mirror image_5 snap 2024-09-14T09:01:21.793 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:01:21.793 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:01:21.793 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_5 2024-09-14T09:01:21.793 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap=snap 2024-09-14T09:01:21.793 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap rm mirror/image_5@snap 2024-09-14T09:01:22.673 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing snap: 100% complete...done. 2024-09-14T09:01:22.681 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image_retry cluster2 mirror image_5 2024-09-14T09:01:22.682 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:01:22.682 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:01:22.682 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_5 2024-09-14T09:01:22.682 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:01:22.682 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0 2024-09-14T09:01:22.683 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_5 2024-09-14T09:01:22.683 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:01:22.683 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:01:22.683 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_5 2024-09-14T09:01:22.683 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_5 2024-09-14T09:01:22.937 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_5 2024-09-14T09:01:23.031 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi164.stderr:2024-09-14T09:01:23.031+0000 7f3e7e175640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5648fee9e000 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-14T09:01:23.032 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi164.stderr:2024-09-14T09:01:23.031+0000 7f3e84982640 -1 rbd::mirror::ImageReplayer: 0x5648feb60780 [3/98c80276-e6b6-4a70-b145-47505e1b7690] operator(): start failed: (85) Interrupted system call should be restarted 2024-09-14T09:01:23.548 INFO:tasks.ceph.cluster1.osd.0.smithi135.stderr:2024-09-14T09:01:23.545+0000 7f31e68c4640 -1 reset not still connected to 0x5623d63185b0 2024-09-14T09:01:23.792 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T09:01:23.795 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:01:23.795 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:01:23.796 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_6 2024-09-14T09:01:23.796 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_snapshot cluster2 mirror image_6 snap 2024-09-14T09:01:23.796 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:01:23.796 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:01:23.796 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_6 2024-09-14T09:01:23.796 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap=snap 2024-09-14T09:01:23.796 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap rm mirror/image_6@snap 2024-09-14T09:01:24.698 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing snap: 100% complete...done. 2024-09-14T09:01:24.711 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image_retry cluster2 mirror image_6 2024-09-14T09:01:24.711 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:01:24.711 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:01:24.711 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_6 2024-09-14T09:01:24.711 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:01:24.711 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0 2024-09-14T09:01:24.712 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_6 2024-09-14T09:01:24.712 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:01:24.712 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:01:24.712 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_6 2024-09-14T09:01:24.712 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_6 2024-09-14T09:01:24.763 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_6 2024-09-14T09:01:24.857 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi164.stderr:2024-09-14T09:01:24.856+0000 7f3e7e976640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5648fee9e820 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-14T09:01:24.857 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi164.stderr:2024-09-14T09:01:24.856+0000 7f3e84982640 -1 rbd::mirror::ImageReplayer: 0x5648feb60f00 [3/82e82fc9-f6e1-4a89-9145-f0e06f4942c1] operator(): start failed: (85) Interrupted system call should be restarted 2024-09-14T09:01:25.712 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T09:01:25.716 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:01:25.716 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:01:25.716 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_7 2024-09-14T09:01:25.716 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_snapshot cluster2 mirror image_7 snap 2024-09-14T09:01:25.716 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:01:25.716 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:01:25.716 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_7 2024-09-14T09:01:25.716 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap=snap 2024-09-14T09:01:25.716 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap rm mirror/image_7@snap 2024-09-14T09:01:26.684 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing snap: 100% complete...done. 2024-09-14T09:01:26.693 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image_retry cluster2 mirror image_7 2024-09-14T09:01:26.693 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:01:26.693 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:01:26.693 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_7 2024-09-14T09:01:26.693 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:01:26.693 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0 2024-09-14T09:01:26.694 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_7 2024-09-14T09:01:26.694 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:01:26.694 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:01:26.695 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_7 2024-09-14T09:01:26.695 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_7 2024-09-14T09:01:26.742 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_7 2024-09-14T09:01:26.844 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi164.stderr:2024-09-14T09:01:26.844+0000 7f3e7e976640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5648fee9fba0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-14T09:01:26.844 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi164.stderr:2024-09-14T09:01:26.844+0000 7f3e84982640 -1 rbd::mirror::ImageReplayer: 0x5648feb61900 [3/46c72cd0-0717-48e2-bd75-e9b03e063bb7] operator(): start failed: (85) Interrupted system call should be restarted 2024-09-14T09:01:27.817 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T09:01:27.820 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:01:27.820 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:01:27.820 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_8 2024-09-14T09:01:27.821 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_snapshot cluster2 mirror image_8 snap 2024-09-14T09:01:27.821 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:01:27.821 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:01:27.821 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_8 2024-09-14T09:01:27.821 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap=snap 2024-09-14T09:01:27.821 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap rm mirror/image_8@snap 2024-09-14T09:01:28.702 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing snap: 100% complete...done. 2024-09-14T09:01:28.713 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image_retry cluster2 mirror image_8 2024-09-14T09:01:28.713 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:01:28.713 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:01:28.713 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_8 2024-09-14T09:01:28.713 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:01:28.713 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0 2024-09-14T09:01:28.714 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_8 2024-09-14T09:01:28.716 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:01:28.716 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:01:28.716 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_8 2024-09-14T09:01:28.716 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_8 2024-09-14T09:01:28.765 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_8 2024-09-14T09:01:28.858 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi164.stderr:2024-09-14T09:01:28.858+0000 7f7e32524640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55b570dbfa00 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-14T09:01:28.858 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi164.stderr:2024-09-14T09:01:28.858+0000 7f7e38d31640 -1 rbd::mirror::ImageReplayer: 0x55b5720e3b80 [3/9b43aad5-e6f1-4a31-a25c-d7df22bd01da] operator(): start failed: (85) Interrupted system call should be restarted 2024-09-14T09:01:29.819 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T09:01:29.822 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:01:29.822 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:01:29.822 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_9 2024-09-14T09:01:29.822 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_snapshot cluster2 mirror image_9 snap 2024-09-14T09:01:29.823 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:01:29.823 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:01:29.823 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_9 2024-09-14T09:01:29.823 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap=snap 2024-09-14T09:01:29.823 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap rm mirror/image_9@snap 2024-09-14T09:01:30.590 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing snap: 100% complete...done. 2024-09-14T09:01:30.601 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image_retry cluster2 mirror image_9 2024-09-14T09:01:30.601 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:01:30.601 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:01:30.601 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_9 2024-09-14T09:01:30.601 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:01:30.601 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0 2024-09-14T09:01:30.602 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_9 2024-09-14T09:01:30.602 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:01:30.602 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:01:30.602 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_9 2024-09-14T09:01:30.602 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_9 2024-09-14T09:01:30.657 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_9 2024-09-14T09:01:30.752 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi164.stderr:2024-09-14T09:01:30.752+0000 7f3e7e175640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5648feb88b60 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-14T09:01:30.752 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi164.stderr:2024-09-14T09:01:30.752+0000 7f3e85183640 -1 rbd::mirror::ImageReplayer: 0x5648feb60500 [3/6b16cbe5-ac26-4754-a383-29c75eb5ef3c] operator(): start failed: (85) Interrupted system call should be restarted 2024-09-14T09:01:31.817 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T09:01:31.821 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:01:31.821 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:01:31.821 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_10 2024-09-14T09:01:31.821 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_snapshot cluster2 mirror image_10 snap 2024-09-14T09:01:31.821 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:01:31.822 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:01:31.822 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_10 2024-09-14T09:01:31.822 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap=snap 2024-09-14T09:01:31.822 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap rm mirror/image_10@snap 2024-09-14T09:01:32.714 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing snap: 100% complete...done. 2024-09-14T09:01:32.728 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image_retry cluster2 mirror image_10 2024-09-14T09:01:32.728 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:01:32.728 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:01:32.728 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_10 2024-09-14T09:01:32.728 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:01:32.728 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0 2024-09-14T09:01:32.729 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_10 2024-09-14T09:01:32.729 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:01:32.729 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:01:32.729 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_10 2024-09-14T09:01:32.730 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_10 2024-09-14T09:01:32.778 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_10 2024-09-14T09:01:32.874 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi164.stderr:2024-09-14T09:01:32.874+0000 7fe44bf2a640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55632c012d00 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-14T09:01:32.874 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi164.stderr:2024-09-14T09:01:32.874+0000 7fe452737640 -1 rbd::mirror::ImageReplayer: 0x55632beb8500 [3/0a80ae74-9f85-4745-bf89-a713c5afc298] operator(): start failed: (85) Interrupted system call should be restarted 2024-09-14T09:01:33.830 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T09:01:33.834 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:01:33.834 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:01:33.834 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_11 2024-09-14T09:01:33.834 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_snapshot cluster2 mirror image_11 snap 2024-09-14T09:01:33.834 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:01:33.834 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:01:33.834 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_11 2024-09-14T09:01:33.834 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap=snap 2024-09-14T09:01:33.834 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap rm mirror/image_11@snap 2024-09-14T09:01:34.715 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing snap: 100% complete...done. 2024-09-14T09:01:34.725 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image_retry cluster2 mirror image_11 2024-09-14T09:01:34.725 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:01:34.725 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:01:34.725 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_11 2024-09-14T09:01:34.725 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:01:34.725 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0 2024-09-14T09:01:34.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_11 2024-09-14T09:01:34.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:01:34.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:01:34.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_11 2024-09-14T09:01:34.727 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_11 2024-09-14T09:01:34.788 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_11 2024-09-14T09:01:34.887 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi164.stderr:2024-09-14T09:01:34.886+0000 7f3754609640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55c94c51f040 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-14T09:01:34.887 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi164.stderr:2024-09-14T09:01:34.886+0000 7f375a615640 -1 rbd::mirror::ImageReplayer: 0x55c94bf16c80 [3/f2624247-ccb3-494f-a736-10ddf00ce6ae] operator(): start failed: (85) Interrupted system call should be restarted 2024-09-14T09:01:35.849 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T09:01:35.853 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:01:35.853 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:01:35.853 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_12 2024-09-14T09:01:35.853 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_snapshot cluster2 mirror image_12 snap 2024-09-14T09:01:35.853 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:01:35.853 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:01:35.853 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_12 2024-09-14T09:01:35.853 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap=snap 2024-09-14T09:01:35.853 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap rm mirror/image_12@snap 2024-09-14T09:01:36.724 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing snap: 100% complete...done. 2024-09-14T09:01:36.733 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image_retry cluster2 mirror image_12 2024-09-14T09:01:36.733 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:01:36.733 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:01:36.734 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_12 2024-09-14T09:01:36.734 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:01:36.734 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0 2024-09-14T09:01:36.734 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_12 2024-09-14T09:01:36.735 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:01:36.735 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:01:36.735 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_12 2024-09-14T09:01:36.735 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_12 2024-09-14T09:01:36.791 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_12 2024-09-14T09:01:36.888 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi164.stderr:2024-09-14T09:01:36.887+0000 7f3753e08640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55c94c51fd40 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-14T09:01:37.861 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T09:01:37.864 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:01:37.864 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:01:37.865 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_13 2024-09-14T09:01:37.865 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_snapshot cluster2 mirror image_13 snap 2024-09-14T09:01:37.865 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:01:37.865 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:01:37.865 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_13 2024-09-14T09:01:37.865 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap=snap 2024-09-14T09:01:37.865 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap rm mirror/image_13@snap 2024-09-14T09:01:38.741 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing snap: 100% complete...done. 2024-09-14T09:01:38.753 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image_retry cluster2 mirror image_13 2024-09-14T09:01:38.753 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:01:38.753 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:01:38.753 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_13 2024-09-14T09:01:38.753 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:01:38.753 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0 2024-09-14T09:01:38.754 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_13 2024-09-14T09:01:38.754 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:01:38.754 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:01:38.754 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_13 2024-09-14T09:01:38.754 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_13 2024-09-14T09:01:38.805 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_13 2024-09-14T09:01:38.900 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi164.stderr:2024-09-14T09:01:38.900+0000 7f7e31d23640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55b570dbeb60 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-14T09:01:38.900 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi164.stderr:2024-09-14T09:01:38.900+0000 7f7e38530640 -1 rbd::mirror::ImageReplayer: 0x55b5720e2280 [3/5dc0c52c-6377-4410-90b1-49344e6d9458] operator(): start failed: (85) Interrupted system call should be restarted 2024-09-14T09:01:39.853 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T09:01:39.857 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:01:39.857 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:01:39.857 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_14 2024-09-14T09:01:39.857 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_snapshot cluster2 mirror image_14 snap 2024-09-14T09:01:39.857 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:01:39.858 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:01:39.858 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_14 2024-09-14T09:01:39.858 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap=snap 2024-09-14T09:01:39.858 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap rm mirror/image_14@snap 2024-09-14T09:01:40.592 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing snap: 100% complete...done. 2024-09-14T09:01:40.603 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image_retry cluster2 mirror image_14 2024-09-14T09:01:40.603 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:01:40.603 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:01:40.603 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_14 2024-09-14T09:01:40.603 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:01:40.603 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0 2024-09-14T09:01:40.604 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_14 2024-09-14T09:01:40.604 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:01:40.604 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:01:40.604 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_14 2024-09-14T09:01:40.605 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_14 2024-09-14T09:01:40.664 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_14 2024-09-14T09:01:40.758 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi164.stderr:2024-09-14T09:01:40.758+0000 7f3e7e175640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5648ff530b60 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-14T09:01:40.758 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi164.stderr:2024-09-14T09:01:40.758+0000 7f3e84982640 -1 rbd::mirror::ImageReplayer: 0x5648feb60c80 [3/d055c61d-091e-4c03-a847-749c1be38f92] operator(): start failed: (85) Interrupted system call should be restarted 2024-09-14T09:01:41.876 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T09:01:41.880 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:01:41.880 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:01:41.880 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_15 2024-09-14T09:01:41.880 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_snapshot cluster2 mirror image_15 snap 2024-09-14T09:01:41.880 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:01:41.880 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:01:41.881 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_15 2024-09-14T09:01:41.881 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap=snap 2024-09-14T09:01:41.881 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap rm mirror/image_15@snap 2024-09-14T09:01:42.748 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing snap: 100% complete...done. 2024-09-14T09:01:42.765 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image_retry cluster2 mirror image_15 2024-09-14T09:01:42.765 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:01:42.766 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:01:42.766 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_15 2024-09-14T09:01:42.766 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:01:42.766 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0 2024-09-14T09:01:42.767 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_15 2024-09-14T09:01:42.767 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:01:42.767 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:01:42.767 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_15 2024-09-14T09:01:42.767 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_15 2024-09-14T09:01:42.818 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_15 2024-09-14T09:01:42.913 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi164.stderr:2024-09-14T09:01:42.913+0000 7f3754609640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55c94b97b040 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-14T09:01:42.914 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi164.stderr:2024-09-14T09:01:42.913+0000 7f375a615640 -1 rbd::mirror::ImageReplayer: 0x55c94ba4db80 [3/b5a7a156-c2fc-47f1-b818-af6739582d53] operator(): start failed: (85) Interrupted system call should be restarted 2024-09-14T09:01:43.881 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T09:01:43.885 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:01:43.885 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:01:43.885 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_16 2024-09-14T09:01:43.886 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_snapshot cluster2 mirror image_16 snap 2024-09-14T09:01:43.886 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:01:43.886 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:01:43.886 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_16 2024-09-14T09:01:43.886 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap=snap 2024-09-14T09:01:43.886 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap rm mirror/image_16@snap 2024-09-14T09:01:44.759 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing snap: 100% complete...done. 2024-09-14T09:01:44.771 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image_retry cluster2 mirror image_16 2024-09-14T09:01:44.771 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:01:44.771 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:01:44.771 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_16 2024-09-14T09:01:44.771 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:01:44.771 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0 2024-09-14T09:01:44.772 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_16 2024-09-14T09:01:44.772 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:01:44.772 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:01:44.772 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_16 2024-09-14T09:01:44.772 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_16 2024-09-14T09:01:44.823 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_16 2024-09-14T09:01:44.926 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi164.stderr:2024-09-14T09:01:44.926+0000 7fe44b729640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55632c012ea0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-14T09:01:44.927 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi164.stderr:2024-09-14T09:01:44.926+0000 7fe452737640 -1 rbd::mirror::ImageReplayer: 0x55632beb9680 [3/095f96c4-d4b7-4fe2-8ba2-0b31704d281f] operator(): start failed: (85) Interrupted system call should be restarted 2024-09-14T09:01:45.881 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T09:01:45.884 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:01:45.885 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:01:45.885 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_17 2024-09-14T09:01:45.885 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_snapshot cluster2 mirror image_17 snap 2024-09-14T09:01:45.885 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:01:45.885 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:01:45.885 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_17 2024-09-14T09:01:45.885 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap=snap 2024-09-14T09:01:45.885 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap rm mirror/image_17@snap 2024-09-14T09:01:46.178 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.5100994294958608) > weight (0.25) 2024-09-14T09:01:46.178 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.9002636180838375) > weight (0.25) 2024-09-14T09:01:46.178 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.42081724647844365) > weight (0.25) 2024-09-14T09:01:46.178 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.3 2024-09-14T09:01:46.179 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Sent signal 15 2024-09-14T09:01:46.179 INFO:tasks.rbd_mirror_thrash:waiting for 7 secs before reviving daemons 2024-09-14T09:01:46.180 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi164.stderr:2024-09-14T09:01:46.179+0000 7f393429d640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.3 (PID: 41285) UID: 1000 2024-09-14T09:01:46.763 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing snap: 100% complete...done. 2024-09-14T09:01:46.771 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image_retry cluster2 mirror image_17 2024-09-14T09:01:46.771 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:01:46.771 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:01:46.772 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_17 2024-09-14T09:01:46.772 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:01:46.772 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0 2024-09-14T09:01:46.772 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_17 2024-09-14T09:01:46.773 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:01:46.773 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:01:46.773 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_17 2024-09-14T09:01:46.773 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_17 2024-09-14T09:01:46.824 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_17 2024-09-14T09:01:46.890 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi164.stderr:2024-09-14T09:01:46.889+0000 7f60ec563640 -1 rbd::mirror::ImageMap: 0x55e319a46120 handle_peer_ack_remove: failed to remove global_image_id=e90f529e-8f18-40e1-aa77-39aa510ca09b 2024-09-14T09:01:47.602 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi164.stderr:2024-09-14T09:01:47.600+0000 7f250ab1a640 -1 rbd::mirror::ImageReplayer: 0x555e4fb31b80 [3/e852057c-0945-4c55-b535-c147c24f9585] start: already running 2024-09-14T09:01:47.878 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T09:01:47.882 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:01:47.882 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:01:47.882 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_18 2024-09-14T09:01:47.882 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_snapshot cluster2 mirror image_18 snap 2024-09-14T09:01:47.882 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:01:47.882 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:01:47.882 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_18 2024-09-14T09:01:47.883 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap=snap 2024-09-14T09:01:47.883 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap rm mirror/image_18@snap 2024-09-14T09:01:48.769 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing snap: 100% complete...done. 2024-09-14T09:01:48.778 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image_retry cluster2 mirror image_18 2024-09-14T09:01:48.778 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:01:48.778 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:01:48.778 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_18 2024-09-14T09:01:48.778 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:01:48.778 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0 2024-09-14T09:01:48.779 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_18 2024-09-14T09:01:48.779 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:01:48.779 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:01:48.780 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_18 2024-09-14T09:01:48.780 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_18 2024-09-14T09:01:48.827 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_18 2024-09-14T09:01:48.890 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi164.stderr:2024-09-14T09:01:48.889+0000 7f60ec563640 -1 rbd::mirror::ImageMap: 0x55e319a46120 handle_peer_ack_remove: failed to remove global_image_id=14306d77-7330-41b5-b70c-85c15b66b18d 2024-09-14T09:01:49.888 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T09:01:49.892 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:01:49.893 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:01:49.893 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_19 2024-09-14T09:01:49.893 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_snapshot cluster2 mirror image_19 snap 2024-09-14T09:01:49.893 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:01:49.893 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:01:49.893 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_19 2024-09-14T09:01:49.894 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap=snap 2024-09-14T09:01:49.894 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap rm mirror/image_19@snap 2024-09-14T09:01:50.593 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing snap: 100% complete...done. 2024-09-14T09:01:50.604 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image_retry cluster2 mirror image_19 2024-09-14T09:01:50.604 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:01:50.604 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:01:50.604 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_19 2024-09-14T09:01:50.604 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:01:50.604 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0 2024-09-14T09:01:50.604 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_19 2024-09-14T09:01:50.604 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:01:50.604 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:01:50.604 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_19 2024-09-14T09:01:50.604 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_19 2024-09-14T09:01:50.657 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_19 2024-09-14T09:01:50.753 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi164.stderr:2024-09-14T09:01:50.751+0000 7f3753e08640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55c94b97b1e0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-14T09:01:50.753 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi164.stderr:2024-09-14T09:01:50.751+0000 7f375ae16640 -1 rbd::mirror::ImageReplayer: 0x55c94bcfbb80 [3/72324075-a734-4269-8742-62ebe1d962b0] operator(): start failed: (85) Interrupted system call should be restarted 2024-09-14T09:01:51.713 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T09:01:51.717 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:01:51.718 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:01:51.718 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_20 2024-09-14T09:01:51.718 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_snapshot cluster2 mirror image_20 snap 2024-09-14T09:01:51.718 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:01:51.718 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:01:51.718 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_20 2024-09-14T09:01:51.718 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap=snap 2024-09-14T09:01:51.718 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap rm mirror/image_20@snap 2024-09-14T09:01:52.782 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing snap: 100% complete...done. 2024-09-14T09:01:52.791 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image_retry cluster2 mirror image_20 2024-09-14T09:01:52.791 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:01:52.792 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:01:52.792 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_20 2024-09-14T09:01:52.792 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:01:52.792 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0 2024-09-14T09:01:52.792 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_20 2024-09-14T09:01:52.792 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:01:52.793 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:01:52.793 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_20 2024-09-14T09:01:52.793 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_20 2024-09-14T09:01:52.843 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_20 2024-09-14T09:01:52.905 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi164.stderr:2024-09-14T09:01:52.904+0000 7f60ec563640 -1 rbd::mirror::ImageMap: 0x55e319a46120 handle_peer_ack_remove: failed to remove global_image_id=f5c8b83b-151b-465c-b8b7-1ea8268e2846 2024-09-14T09:01:53.180 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.3 2024-09-14T09:01:53.180 INFO:teuthology.orchestra.run:waiting for 600 2024-09-14T09:01:53.181 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.3 2024-09-14T09:01:53.181 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Restarting daemon 2024-09-14T09:01:53.182 DEBUG:teuthology.orchestra.run.smithi164:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.3 2024-09-14T09:01:53.185 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Started 2024-09-14T09:01:53.185 INFO:tasks.rbd_mirror_thrash:waiting for 79 secs before thrashing 2024-09-14T09:01:53.897 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T09:01:53.901 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:01:53.901 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:01:53.901 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_21 2024-09-14T09:01:53.901 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_snapshot cluster2 mirror image_21 snap 2024-09-14T09:01:53.901 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:01:53.901 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:01:53.901 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_21 2024-09-14T09:01:53.901 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap=snap 2024-09-14T09:01:53.901 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap rm mirror/image_21@snap 2024-09-14T09:01:54.777 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing snap: 100% complete...done. 2024-09-14T09:01:54.788 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image_retry cluster2 mirror image_21 2024-09-14T09:01:54.788 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:01:54.788 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:01:54.788 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_21 2024-09-14T09:01:54.788 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:01:54.788 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0 2024-09-14T09:01:54.789 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_21 2024-09-14T09:01:54.789 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:01:54.789 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:01:54.789 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_21 2024-09-14T09:01:54.789 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_21 2024-09-14T09:01:54.838 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_21 2024-09-14T09:01:54.940 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi164.stderr:2024-09-14T09:01:54.939+0000 7f7e32524640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55b570dbf520 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-14T09:01:54.940 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi164.stderr:2024-09-14T09:01:54.939+0000 7f7e38530640 -1 rbd::mirror::ImageReplayer: 0x55b5720e2780 [3/75208eea-1f60-4048-a017-3885fa315408] operator(): start failed: (85) Interrupted system call should be restarted 2024-09-14T09:01:55.898 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T09:01:55.902 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:01:55.902 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:01:55.902 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_22 2024-09-14T09:01:55.902 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_snapshot cluster2 mirror image_22 snap 2024-09-14T09:01:55.902 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:01:55.902 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:01:55.903 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_22 2024-09-14T09:01:55.903 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap=snap 2024-09-14T09:01:55.903 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap rm mirror/image_22@snap 2024-09-14T09:01:56.793 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing snap: 100% complete...done. 2024-09-14T09:01:56.808 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image_retry cluster2 mirror image_22 2024-09-14T09:01:56.808 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:01:56.808 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:01:56.808 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_22 2024-09-14T09:01:56.808 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:01:56.808 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0 2024-09-14T09:01:56.809 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_22 2024-09-14T09:01:56.809 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:01:56.809 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:01:56.809 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_22 2024-09-14T09:01:56.810 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_22 2024-09-14T09:01:56.862 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_22 2024-09-14T09:01:56.958 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi164.stderr:2024-09-14T09:01:56.957+0000 7fe44b729640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55632bea0b60 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-14T09:01:56.958 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi164.stderr:2024-09-14T09:01:56.957+0000 7fe451f36640 -1 rbd::mirror::ImageReplayer: 0x55632beb8f00 [3/83c2e42b-5fd9-42ba-b3f2-2e9c690a99bf] operator(): start failed: (85) Interrupted system call should be restarted 2024-09-14T09:01:57.918 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T09:01:57.922 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:01:57.922 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:01:57.922 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_23 2024-09-14T09:01:57.922 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_snapshot cluster2 mirror image_23 snap 2024-09-14T09:01:57.923 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:01:57.923 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:01:57.923 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_23 2024-09-14T09:01:57.923 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap=snap 2024-09-14T09:01:57.923 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap rm mirror/image_23@snap 2024-09-14T09:01:58.804 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing snap: 100% complete...done. 2024-09-14T09:01:58.816 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image_retry cluster2 mirror image_23 2024-09-14T09:01:58.817 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:01:58.817 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:01:58.817 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_23 2024-09-14T09:01:58.817 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:01:58.817 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0 2024-09-14T09:01:58.818 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_23 2024-09-14T09:01:58.818 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:01:58.818 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:01:58.818 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_23 2024-09-14T09:01:58.818 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_23 2024-09-14T09:01:58.874 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_23 2024-09-14T09:01:58.934 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi164.stderr:2024-09-14T09:01:58.934+0000 7f60ec563640 -1 rbd::mirror::ImageMap: 0x55e319a46120 handle_peer_ack_remove: failed to remove global_image_id=b97e8af3-4a7b-444b-9f49-8151fd4bf536 2024-09-14T09:01:59.932 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T09:01:59.935 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:01:59.936 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:01:59.936 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_24 2024-09-14T09:01:59.936 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_snapshot cluster2 mirror image_24 snap 2024-09-14T09:01:59.936 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:01:59.936 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:01:59.936 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_24 2024-09-14T09:01:59.936 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap=snap 2024-09-14T09:01:59.936 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap rm mirror/image_24@snap 2024-09-14T09:02:00.596 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing snap: 100% complete...done. 2024-09-14T09:02:00.609 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image_retry cluster2 mirror image_24 2024-09-14T09:02:00.611 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:02:00.611 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:00.611 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_24 2024-09-14T09:02:00.611 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:02:00.611 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0 2024-09-14T09:02:00.613 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_24 2024-09-14T09:02:00.613 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:02:00.614 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:00.614 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_24 2024-09-14T09:02:00.614 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_24 2024-09-14T09:02:00.667 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_24 2024-09-14T09:02:00.765 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi164.stderr:2024-09-14T09:02:00.764+0000 7f3753e08640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55c94bd9da00 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-14T09:02:00.765 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi164.stderr:2024-09-14T09:02:00.764+0000 7f375a615640 -1 rbd::mirror::ImageReplayer: 0x55c94ba4d680 [3/24c48f34-7f6e-4594-86fc-0a765052746d] operator(): start failed: (85) Interrupted system call should be restarted 2024-09-14T09:02:01.718 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T09:02:01.721 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:02:01.721 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:02:01.722 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_25 2024-09-14T09:02:01.722 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_snapshot cluster2 mirror image_25 snap 2024-09-14T09:02:01.722 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:02:01.722 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:01.722 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_25 2024-09-14T09:02:01.722 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap=snap 2024-09-14T09:02:01.722 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap rm mirror/image_25@snap 2024-09-14T09:02:01.952 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.7384583696680828) > weight (0.25) 2024-09-14T09:02:01.953 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.6641666807865786) > weight (0.25) 2024-09-14T09:02:01.953 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.8256493286700769) > weight (0.25) 2024-09-14T09:02:01.953 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.3 with skip (0.6450934432697467) > weight (0.25) 2024-09-14T09:02:01.953 INFO:tasks.rbd_mirror_thrash:waiting for 93 secs before thrashing 2024-09-14T09:02:02.600 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing snap: 100% complete...done. 2024-09-14T09:02:02.612 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image_retry cluster2 mirror image_25 2024-09-14T09:02:02.612 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:02:02.612 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:02.612 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_25 2024-09-14T09:02:02.612 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:02:02.612 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0 2024-09-14T09:02:02.617 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_25 2024-09-14T09:02:02.617 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:02:02.617 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:02.618 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_25 2024-09-14T09:02:02.618 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_25 2024-09-14T09:02:02.666 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_25 2024-09-14T09:02:02.963 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi164.stderr:2024-09-14T09:02:02.962+0000 7fe44bf2a640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55632c013040 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-14T09:02:02.963 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi164.stderr:2024-09-14T09:02:02.962+0000 7fe452737640 -1 rbd::mirror::ImageReplayer: 0x55632beb8a00 [3/3d365e96-a01e-4587-97b2-7c3efff31e05] operator(): start failed: (85) Interrupted system call should be restarted 2024-09-14T09:02:03.722 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T09:02:03.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:02:03.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:02:03.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_26 2024-09-14T09:02:03.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_snapshot cluster2 mirror image_26 snap 2024-09-14T09:02:03.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:02:03.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:03.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_26 2024-09-14T09:02:03.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap=snap 2024-09-14T09:02:03.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap rm mirror/image_26@snap 2024-09-14T09:02:04.604 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing snap: 100% complete...done. 2024-09-14T09:02:04.616 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image_retry cluster2 mirror image_26 2024-09-14T09:02:04.616 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:02:04.616 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:04.617 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_26 2024-09-14T09:02:04.617 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:02:04.617 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0 2024-09-14T09:02:04.617 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_26 2024-09-14T09:02:04.618 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:02:04.618 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:04.618 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_26 2024-09-14T09:02:04.618 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_26 2024-09-14T09:02:04.669 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_26 2024-09-14T09:02:04.770 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi164.stderr:2024-09-14T09:02:04.770+0000 7f3e7e175640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5648feb88340 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-14T09:02:04.770 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi164.stderr:2024-09-14T09:02:04.770+0000 7f3e85183640 -1 rbd::mirror::ImageReplayer: 0x5648feb61680 [3/4ec19e01-86f0-46e8-9aed-ae17d2da69c7] operator(): start failed: (85) Interrupted system call should be restarted 2024-09-14T09:02:05.746 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T09:02:05.750 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:02:05.750 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:02:05.750 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_27 2024-09-14T09:02:05.750 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_snapshot cluster2 mirror image_27 snap 2024-09-14T09:02:05.750 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:02:05.750 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:05.750 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_27 2024-09-14T09:02:05.750 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap=snap 2024-09-14T09:02:05.750 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap rm mirror/image_27@snap 2024-09-14T09:02:06.627 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing snap: 100% complete...done. 2024-09-14T09:02:06.638 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image_retry cluster2 mirror image_27 2024-09-14T09:02:06.638 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:02:06.638 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:06.638 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_27 2024-09-14T09:02:06.638 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:02:06.638 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0 2024-09-14T09:02:06.639 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_27 2024-09-14T09:02:06.639 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:02:06.640 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:06.640 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_27 2024-09-14T09:02:06.640 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_27 2024-09-14T09:02:06.899 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_27 2024-09-14T09:02:07.005 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi164.stderr:2024-09-14T09:02:07.005+0000 7f3e7e175640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5648feb89ba0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-14T09:02:07.005 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi164.stderr:2024-09-14T09:02:07.005+0000 7f3e85183640 -1 rbd::mirror::ImageReplayer: 0x5648feb61b80 [3/a44041c2-b983-4eef-b3f2-aec32b4ab580] operator(): start failed: (85) Interrupted system call should be restarted 2024-09-14T09:02:07.759 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T09:02:07.762 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:02:07.762 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:02:07.763 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_28 2024-09-14T09:02:07.763 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_snapshot cluster2 mirror image_28 snap 2024-09-14T09:02:07.763 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:02:07.763 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:07.763 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_28 2024-09-14T09:02:07.763 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap=snap 2024-09-14T09:02:07.763 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap rm mirror/image_28@snap 2024-09-14T09:02:08.631 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing snap: 100% complete...done. 2024-09-14T09:02:08.640 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image_retry cluster2 mirror image_28 2024-09-14T09:02:08.640 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:02:08.640 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:08.640 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_28 2024-09-14T09:02:08.640 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:02:08.641 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0 2024-09-14T09:02:08.642 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_28 2024-09-14T09:02:08.642 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:02:08.642 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:08.642 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_28 2024-09-14T09:02:08.642 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_28 2024-09-14T09:02:08.694 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_28 2024-09-14T09:02:08.759 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi164.stderr:2024-09-14T09:02:08.759+0000 7f60ebd62640 -1 rbd::mirror::ImageMap: 0x55e319a46120 handle_peer_ack_remove: failed to remove global_image_id=e5b5824c-8477-465c-bd16-4b286b9a4cc9 2024-09-14T09:02:09.768 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T09:02:09.772 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:02:09.772 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:02:09.772 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_29 2024-09-14T09:02:09.772 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_snapshot cluster2 mirror image_29 snap 2024-09-14T09:02:09.772 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:02:09.772 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:09.772 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_29 2024-09-14T09:02:09.772 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap=snap 2024-09-14T09:02:09.772 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap rm mirror/image_29@snap 2024-09-14T09:02:10.597 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing snap: 100% complete...done. 2024-09-14T09:02:10.606 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image_retry cluster2 mirror image_29 2024-09-14T09:02:10.606 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:02:10.606 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:10.607 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_29 2024-09-14T09:02:10.607 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:02:10.607 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0 2024-09-14T09:02:10.608 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_29 2024-09-14T09:02:10.608 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:02:10.608 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:10.608 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_29 2024-09-14T09:02:10.608 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_29 2024-09-14T09:02:10.656 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_29 2024-09-14T09:02:10.766 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi164.stderr:2024-09-14T09:02:10.765+0000 7fe44b729640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55632bea1ba0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-14T09:02:10.766 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi164.stderr:2024-09-14T09:02:10.765+0000 7fe451f36640 -1 rbd::mirror::ImageReplayer: 0x55632beb9180 [3/50c4f65a-9027-4112-bc18-18bc2d737492] operator(): start failed: (85) Interrupted system call should be restarted 2024-09-14T09:02:11.760 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T09:02:11.764 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:02:11.764 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:02:11.764 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_30 2024-09-14T09:02:11.764 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_snapshot cluster2 mirror image_30 snap 2024-09-14T09:02:11.764 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:02:11.764 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:11.764 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_30 2024-09-14T09:02:11.765 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap=snap 2024-09-14T09:02:11.765 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap rm mirror/image_30@snap 2024-09-14T09:02:12.640 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing snap: 100% complete...done. 2024-09-14T09:02:12.652 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image_retry cluster2 mirror image_30 2024-09-14T09:02:12.652 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:02:12.652 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:12.653 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_30 2024-09-14T09:02:12.653 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:02:12.653 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0 2024-09-14T09:02:12.654 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_30 2024-09-14T09:02:12.654 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:02:12.654 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:12.654 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_30 2024-09-14T09:02:12.654 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_30 2024-09-14T09:02:12.705 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_30 2024-09-14T09:02:12.811 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi164.stderr:2024-09-14T09:02:12.811+0000 7f3e7e976640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5648feb88d00 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-14T09:02:12.811 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi164.stderr:2024-09-14T09:02:12.811+0000 7f3e85183640 -1 rbd::mirror::ImageReplayer: 0x5648feb61400 [3/4e345c40-e13b-4117-8365-22395b48e4b8] operator(): start failed: (85) Interrupted system call should be restarted 2024-09-14T09:02:13.769 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T09:02:13.772 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:02:13.772 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:02:13.772 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_31 2024-09-14T09:02:13.772 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_snapshot cluster2 mirror image_31 snap 2024-09-14T09:02:13.772 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:02:13.773 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:13.773 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_31 2024-09-14T09:02:13.773 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap=snap 2024-09-14T09:02:13.773 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap rm mirror/image_31@snap 2024-09-14T09:02:14.646 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing snap: 100% complete...done. 2024-09-14T09:02:14.654 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image_retry cluster2 mirror image_31 2024-09-14T09:02:14.655 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:02:14.655 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:14.655 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_31 2024-09-14T09:02:14.655 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:02:14.655 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0 2024-09-14T09:02:14.656 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_31 2024-09-14T09:02:14.656 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:02:14.656 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:14.656 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_31 2024-09-14T09:02:14.656 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_31 2024-09-14T09:02:14.706 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_31 2024-09-14T09:02:14.767 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi164.stderr:2024-09-14T09:02:14.767+0000 7f60ec563640 -1 rbd::mirror::ImageMap: 0x55e319a46120 handle_peer_ack_remove: failed to remove global_image_id=ca4b7bbf-4e51-4087-ac80-e20897556ab1 2024-09-14T09:02:15.771 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T09:02:15.775 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:02:15.775 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:02:15.775 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_32 2024-09-14T09:02:15.775 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_snapshot cluster2 mirror image_32 snap 2024-09-14T09:02:15.776 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:02:15.776 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:15.776 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_32 2024-09-14T09:02:15.776 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap=snap 2024-09-14T09:02:15.776 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap rm mirror/image_32@snap 2024-09-14T09:02:16.652 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing snap: 100% complete...done. 2024-09-14T09:02:16.664 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image_retry cluster2 mirror image_32 2024-09-14T09:02:16.664 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:02:16.664 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:16.664 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_32 2024-09-14T09:02:16.664 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:02:16.664 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0 2024-09-14T09:02:16.665 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_32 2024-09-14T09:02:16.665 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:02:16.665 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:16.665 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_32 2024-09-14T09:02:16.665 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_32 2024-09-14T09:02:16.721 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_32 2024-09-14T09:02:16.782 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi164.stderr:2024-09-14T09:02:16.781+0000 7f60ec563640 -1 rbd::mirror::ImageMap: 0x55e319a46120 handle_peer_ack_remove: failed to remove global_image_id=7da0399c-f6eb-473e-a82d-0b57a2ea8a70 2024-09-14T09:02:17.785 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T09:02:17.789 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:02:17.789 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:02:17.789 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_33 2024-09-14T09:02:17.789 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_snapshot cluster2 mirror image_33 snap 2024-09-14T09:02:17.789 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:02:17.789 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:17.789 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_33 2024-09-14T09:02:17.789 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap=snap 2024-09-14T09:02:17.789 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap rm mirror/image_33@snap 2024-09-14T09:02:18.667 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing snap: 100% complete...done. 2024-09-14T09:02:18.676 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image_retry cluster2 mirror image_33 2024-09-14T09:02:18.676 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:02:18.676 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:18.676 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_33 2024-09-14T09:02:18.676 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:02:18.676 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0 2024-09-14T09:02:18.677 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_33 2024-09-14T09:02:18.677 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:02:18.677 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:18.677 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_33 2024-09-14T09:02:18.677 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_33 2024-09-14T09:02:18.734 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_33 2024-09-14T09:02:18.833 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi164.stderr:2024-09-14T09:02:18.832+0000 7f3754609640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55c94ba131e0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-14T09:02:18.833 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi164.stderr:2024-09-14T09:02:18.832+0000 7f375ae16640 -1 rbd::mirror::ImageReplayer: 0x55c94ba4d400 [3/baaf07f1-8725-4ffd-a1a5-efd3fb31c8b6] operator(): start failed: (85) Interrupted system call should be restarted 2024-09-14T09:02:19.815 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T09:02:19.819 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:02:19.819 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:02:19.819 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_34 2024-09-14T09:02:19.820 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_snapshot cluster2 mirror image_34 snap 2024-09-14T09:02:19.820 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:02:19.820 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:19.820 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_34 2024-09-14T09:02:19.820 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap=snap 2024-09-14T09:02:19.820 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap rm mirror/image_34@snap 2024-09-14T09:02:20.604 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing snap: 100% complete...done. 2024-09-14T09:02:20.615 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image_retry cluster2 mirror image_34 2024-09-14T09:02:20.616 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:02:20.616 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:20.616 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_34 2024-09-14T09:02:20.616 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:02:20.616 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0 2024-09-14T09:02:20.617 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_34 2024-09-14T09:02:20.617 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:02:20.617 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:20.617 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_34 2024-09-14T09:02:20.617 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_34 2024-09-14T09:02:20.668 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_34 2024-09-14T09:02:20.769 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi164.stderr:2024-09-14T09:02:20.768+0000 7f7e32524640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55b570dbf040 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-14T09:02:20.769 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi164.stderr:2024-09-14T09:02:20.768+0000 7f7e38530640 -1 rbd::mirror::ImageReplayer: 0x55b57212e000 [3/ec5aa9c3-0e81-40f0-a328-f1ef0300304d] operator(): start failed: (85) Interrupted system call should be restarted 2024-09-14T09:02:21.795 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T09:02:21.799 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:02:21.799 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:02:21.799 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_35 2024-09-14T09:02:21.799 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_snapshot cluster2 mirror image_35 snap 2024-09-14T09:02:21.799 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:02:21.799 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:21.799 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_35 2024-09-14T09:02:21.799 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap=snap 2024-09-14T09:02:21.799 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap rm mirror/image_35@snap 2024-09-14T09:02:22.678 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing snap: 100% complete...done. 2024-09-14T09:02:22.692 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image_retry cluster2 mirror image_35 2024-09-14T09:02:22.692 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:02:22.692 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:22.692 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_35 2024-09-14T09:02:22.692 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:02:22.692 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0 2024-09-14T09:02:22.693 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_35 2024-09-14T09:02:22.693 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:02:22.693 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:22.693 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_35 2024-09-14T09:02:22.693 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_35 2024-09-14T09:02:22.744 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_35 2024-09-14T09:02:22.846 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi164.stderr:2024-09-14T09:02:22.846+0000 7f7e32524640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55b571b1aea0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-14T09:02:22.846 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi164.stderr:2024-09-14T09:02:22.846+0000 7f7e38d31640 -1 rbd::mirror::ImageReplayer: 0x55b5720e3180 [3/285cc24e-11c6-46a3-8120-f1a409abec99] operator(): start failed: (85) Interrupted system call should be restarted 2024-09-14T09:02:23.808 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T09:02:23.812 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:02:23.812 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:02:23.812 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_36 2024-09-14T09:02:23.812 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_snapshot cluster2 mirror image_36 snap 2024-09-14T09:02:23.812 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:02:23.812 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:23.812 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_36 2024-09-14T09:02:23.812 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap=snap 2024-09-14T09:02:23.813 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap rm mirror/image_36@snap 2024-09-14T09:02:24.692 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing snap: 100% complete...done. 2024-09-14T09:02:24.702 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image_retry cluster2 mirror image_36 2024-09-14T09:02:24.702 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:02:24.703 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:24.703 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_36 2024-09-14T09:02:24.703 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:02:24.703 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0 2024-09-14T09:02:24.704 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_36 2024-09-14T09:02:24.705 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:02:24.705 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:24.705 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_36 2024-09-14T09:02:24.705 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_36 2024-09-14T09:02:24.760 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_36 2024-09-14T09:02:24.857 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi164.stderr:2024-09-14T09:02:24.857+0000 7f7e32524640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55b571b1ad00 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-14T09:02:24.857 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi164.stderr:2024-09-14T09:02:24.857+0000 7f7e38d31640 -1 rbd::mirror::ImageReplayer: 0x55b5720e2f00 [3/1e059464-8982-44b9-a52b-107ceeccb93e] operator(): start failed: (85) Interrupted system call should be restarted 2024-09-14T09:02:25.815 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T09:02:25.818 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:02:25.819 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:02:25.819 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_37 2024-09-14T09:02:25.819 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_snapshot cluster2 mirror image_37 snap 2024-09-14T09:02:25.819 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:02:25.819 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:25.819 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_37 2024-09-14T09:02:25.819 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap=snap 2024-09-14T09:02:25.819 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap rm mirror/image_37@snap 2024-09-14T09:02:26.699 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing snap: 100% complete...done. 2024-09-14T09:02:26.712 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image_retry cluster2 mirror image_37 2024-09-14T09:02:26.712 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:02:26.712 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:26.712 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_37 2024-09-14T09:02:26.713 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:02:26.713 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0 2024-09-14T09:02:26.716 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_37 2024-09-14T09:02:26.716 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:02:26.716 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:26.716 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_37 2024-09-14T09:02:26.716 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_37 2024-09-14T09:02:26.764 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_37 2024-09-14T09:02:26.855 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi164.stderr:2024-09-14T09:02:26.855+0000 7f3753e08640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55c94be29860 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-14T09:02:26.855 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi164.stderr:2024-09-14T09:02:26.855+0000 7f375ae16640 -1 rbd::mirror::ImageReplayer: 0x55c94bcfa000 [3/f4bb57da-c08d-42af-8cd0-9868b7375dfb] operator(): start failed: (85) Interrupted system call should be restarted 2024-09-14T09:02:27.812 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T09:02:27.816 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:02:27.816 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:02:27.816 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_38 2024-09-14T09:02:27.817 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_snapshot cluster2 mirror image_38 snap 2024-09-14T09:02:27.817 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:02:27.817 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:27.817 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_38 2024-09-14T09:02:27.817 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap=snap 2024-09-14T09:02:27.817 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap rm mirror/image_38@snap 2024-09-14T09:02:28.714 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing snap: 100% complete...done. 2024-09-14T09:02:28.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image_retry cluster2 mirror image_38 2024-09-14T09:02:28.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:02:28.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:28.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_38 2024-09-14T09:02:28.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:02:28.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0 2024-09-14T09:02:28.727 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_38 2024-09-14T09:02:28.727 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:02:28.727 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:28.727 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_38 2024-09-14T09:02:28.727 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_38 2024-09-14T09:02:28.779 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_38 2024-09-14T09:02:28.875 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi164.stderr:2024-09-14T09:02:28.875+0000 7fe44b729640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55632b5fa680 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-14T09:02:28.875 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi164.stderr:2024-09-14T09:02:28.875+0000 7fe451f36640 -1 rbd::mirror::ImageReplayer: 0x55632beb8c80 [3/56c69995-3ca1-4975-bc59-f14d91322ad4] operator(): start failed: (85) Interrupted system call should be restarted 2024-09-14T09:02:29.827 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T09:02:29.831 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:02:29.831 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:02:29.831 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_39 2024-09-14T09:02:29.831 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_snapshot cluster2 mirror image_39 snap 2024-09-14T09:02:29.832 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:02:29.832 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:29.832 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_39 2024-09-14T09:02:29.832 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap=snap 2024-09-14T09:02:29.832 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap rm mirror/image_39@snap 2024-09-14T09:02:30.598 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing snap: 100% complete...done. 2024-09-14T09:02:30.613 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image_retry cluster2 mirror image_39 2024-09-14T09:02:30.613 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:02:30.613 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:30.613 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_39 2024-09-14T09:02:30.613 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:02:30.613 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0 2024-09-14T09:02:30.614 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_39 2024-09-14T09:02:30.614 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:02:30.614 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:30.614 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_39 2024-09-14T09:02:30.614 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_39 2024-09-14T09:02:30.868 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_39 2024-09-14T09:02:30.969 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi164.stderr:2024-09-14T09:02:30.969+0000 7f3754609640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55c94be296c0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-14T09:02:30.970 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi164.stderr:2024-09-14T09:02:30.969+0000 7f375a615640 -1 rbd::mirror::ImageReplayer: 0x55c94bcfaf00 [3/e2dde19c-e7be-4645-8807-0ea1479cbd41] operator(): start failed: (85) Interrupted system call should be restarted 2024-09-14T09:02:31.825 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T09:02:31.829 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:02:31.829 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:02:31.829 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_40 2024-09-14T09:02:31.829 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_snapshot cluster2 mirror image_40 snap 2024-09-14T09:02:31.829 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:02:31.829 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:31.832 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_40 2024-09-14T09:02:31.832 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap=snap 2024-09-14T09:02:31.832 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap rm mirror/image_40@snap 2024-09-14T09:02:32.713 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing snap: 100% complete...done. 2024-09-14T09:02:32.724 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image_retry cluster2 mirror image_40 2024-09-14T09:02:32.724 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:02:32.724 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:32.724 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_40 2024-09-14T09:02:32.724 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:02:32.724 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0 2024-09-14T09:02:32.725 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_40 2024-09-14T09:02:32.725 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:02:32.725 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:32.725 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_40 2024-09-14T09:02:32.725 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_40 2024-09-14T09:02:32.778 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_40 2024-09-14T09:02:32.876 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi164.stderr:2024-09-14T09:02:32.875+0000 7f7e31d23640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55b570dbf860 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-14T09:02:32.876 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi164.stderr:2024-09-14T09:02:32.875+0000 7f7e38530640 -1 rbd::mirror::ImageReplayer: 0x55b5720e3400 [3/3327bc1e-5162-4a18-8a41-c4a1c978f5cf] operator(): start failed: (85) Interrupted system call should be restarted 2024-09-14T09:02:33.847 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T09:02:33.851 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:02:33.851 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:02:33.851 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_41 2024-09-14T09:02:33.851 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_snapshot cluster2 mirror image_41 snap 2024-09-14T09:02:33.851 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:02:33.851 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:33.851 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_41 2024-09-14T09:02:33.851 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap=snap 2024-09-14T09:02:33.851 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap rm mirror/image_41@snap 2024-09-14T09:02:34.729 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing snap: 100% complete...done. 2024-09-14T09:02:34.738 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image_retry cluster2 mirror image_41 2024-09-14T09:02:34.738 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:02:34.738 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:34.739 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_41 2024-09-14T09:02:34.739 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:02:34.739 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0 2024-09-14T09:02:34.739 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_41 2024-09-14T09:02:34.740 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:02:34.740 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:34.740 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_41 2024-09-14T09:02:34.740 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_41 2024-09-14T09:02:34.790 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_41 2024-09-14T09:02:34.884 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi164.stderr:2024-09-14T09:02:34.884+0000 7f3e7e976640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5648feb89ba0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-14T09:02:34.884 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi164.stderr:2024-09-14T09:02:34.884+0000 7f3e85183640 -1 rbd::mirror::ImageReplayer: 0x5648feb60280 [3/b734de70-db13-448a-a651-6020a282fb17] operator(): start failed: (85) Interrupted system call should be restarted 2024-09-14T09:02:35.841 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T09:02:35.845 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:02:35.845 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:02:35.845 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_42 2024-09-14T09:02:35.845 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_snapshot cluster2 mirror image_42 snap 2024-09-14T09:02:35.845 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:02:35.845 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:35.846 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_42 2024-09-14T09:02:35.846 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap=snap 2024-09-14T09:02:35.846 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap rm mirror/image_42@snap 2024-09-14T09:02:36.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing snap: 100% complete...done. 2024-09-14T09:02:36.735 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image_retry cluster2 mirror image_42 2024-09-14T09:02:36.735 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:02:36.735 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:36.735 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_42 2024-09-14T09:02:36.736 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:02:36.736 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0 2024-09-14T09:02:36.736 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_42 2024-09-14T09:02:36.736 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:02:36.736 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:36.737 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_42 2024-09-14T09:02:36.737 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_42 2024-09-14T09:02:36.793 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_42 2024-09-14T09:02:36.890 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi164.stderr:2024-09-14T09:02:36.890+0000 7fe44bf2a640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55632bea04e0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-14T09:02:36.891 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi164.stderr:2024-09-14T09:02:36.890+0000 7fe451f36640 -1 rbd::mirror::ImageReplayer: 0x55632beb8280 [3/071d6746-d644-418c-9496-15e5e66ef4f1] operator(): start failed: (85) Interrupted system call should be restarted 2024-09-14T09:02:37.853 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T09:02:37.857 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:02:37.857 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:02:37.857 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_43 2024-09-14T09:02:37.857 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_snapshot cluster2 mirror image_43 snap 2024-09-14T09:02:37.857 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:02:37.857 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:37.857 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_43 2024-09-14T09:02:37.857 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap=snap 2024-09-14T09:02:37.857 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap rm mirror/image_43@snap 2024-09-14T09:02:38.745 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing snap: 100% complete...done. 2024-09-14T09:02:38.754 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image_retry cluster2 mirror image_43 2024-09-14T09:02:38.754 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:02:38.754 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:38.754 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_43 2024-09-14T09:02:38.754 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:02:38.754 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0 2024-09-14T09:02:38.755 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_43 2024-09-14T09:02:38.755 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:02:38.755 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:38.755 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_43 2024-09-14T09:02:38.755 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_43 2024-09-14T09:02:38.807 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_43 2024-09-14T09:02:38.911 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi164.stderr:2024-09-14T09:02:38.911+0000 7f7e31d23640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55b570dbf040 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-14T09:02:38.911 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi164.stderr:2024-09-14T09:02:38.911+0000 7f7e38d31640 -1 rbd::mirror::ImageReplayer: 0x55b5720e3680 [3/3589cded-f286-4a62-a351-1f0a15e1b107] operator(): start failed: (85) Interrupted system call should be restarted 2024-09-14T09:02:39.868 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T09:02:39.872 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:02:39.872 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:02:39.872 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_44 2024-09-14T09:02:39.873 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_snapshot cluster2 mirror image_44 snap 2024-09-14T09:02:39.873 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:02:39.873 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:39.873 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_44 2024-09-14T09:02:39.873 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap=snap 2024-09-14T09:02:39.873 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap rm mirror/image_44@snap 2024-09-14T09:02:40.611 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing snap: 100% complete...done. 2024-09-14T09:02:40.623 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image_retry cluster2 mirror image_44 2024-09-14T09:02:40.623 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:02:40.624 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:40.624 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_44 2024-09-14T09:02:40.624 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:02:40.624 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0 2024-09-14T09:02:40.625 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_44 2024-09-14T09:02:40.625 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:02:40.625 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:40.625 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_44 2024-09-14T09:02:40.625 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_44 2024-09-14T09:02:40.676 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_44 2024-09-14T09:02:40.772 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi164.stderr:2024-09-14T09:02:40.772+0000 7f3e7e175640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5648feb891e0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-14T09:02:40.773 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi164.stderr:2024-09-14T09:02:40.772+0000 7f3e85183640 -1 rbd::mirror::ImageReplayer: 0x5648fed6c000 [3/484caad3-31ac-439e-a6d6-4a11118a3aed] operator(): start failed: (85) Interrupted system call should be restarted 2024-09-14T09:02:41.872 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T09:02:41.876 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:02:41.876 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:02:41.876 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_45 2024-09-14T09:02:41.876 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_snapshot cluster2 mirror image_45 snap 2024-09-14T09:02:41.876 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:02:41.876 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:41.877 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_45 2024-09-14T09:02:41.877 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap=snap 2024-09-14T09:02:41.877 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap rm mirror/image_45@snap 2024-09-14T09:02:42.753 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing snap: 100% complete...done. 2024-09-14T09:02:42.761 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image_retry cluster2 mirror image_45 2024-09-14T09:02:42.762 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:02:42.762 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:42.762 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_45 2024-09-14T09:02:42.762 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:02:42.762 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0 2024-09-14T09:02:42.762 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_45 2024-09-14T09:02:42.763 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:02:42.763 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:42.763 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_45 2024-09-14T09:02:42.763 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_45 2024-09-14T09:02:42.819 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_45 2024-09-14T09:02:42.913 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi164.stderr:2024-09-14T09:02:42.913+0000 7f3753e08640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55c94be28000 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-14T09:02:42.913 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi164.stderr:2024-09-14T09:02:42.913+0000 7f375ae16640 -1 rbd::mirror::ImageReplayer: 0x55c94ba4cc80 [3/c2a67c35-feb8-43a1-95f7-a8a88b0481e9] operator(): start failed: (85) Interrupted system call should be restarted 2024-09-14T09:02:43.874 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T09:02:43.877 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:02:43.877 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:02:43.877 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_46 2024-09-14T09:02:43.877 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_snapshot cluster2 mirror image_46 snap 2024-09-14T09:02:43.877 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:02:43.877 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:43.877 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_46 2024-09-14T09:02:43.877 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap=snap 2024-09-14T09:02:43.878 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap rm mirror/image_46@snap 2024-09-14T09:02:44.764 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing snap: 100% complete...done. 2024-09-14T09:02:44.776 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image_retry cluster2 mirror image_46 2024-09-14T09:02:44.777 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:02:44.777 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:44.777 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_46 2024-09-14T09:02:44.777 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:02:44.777 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0 2024-09-14T09:02:44.780 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_46 2024-09-14T09:02:44.781 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:02:44.781 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:44.781 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_46 2024-09-14T09:02:44.781 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_46 2024-09-14T09:02:44.837 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_46 2024-09-14T09:02:44.940 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi164.stderr:2024-09-14T09:02:44.940+0000 7fe44bf2a640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55632b5fad00 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-14T09:02:44.940 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi164.stderr:2024-09-14T09:02:44.940+0000 7fe452737640 -1 rbd::mirror::ImageReplayer: 0x55632bf46f00 [3/0f89d25d-4adb-4110-bda2-74b7704ad44b] operator(): start failed: (85) Interrupted system call should be restarted 2024-09-14T09:02:45.887 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T09:02:45.891 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:02:45.891 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:02:45.891 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_47 2024-09-14T09:02:45.891 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_snapshot cluster2 mirror image_47 snap 2024-09-14T09:02:45.891 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:02:45.891 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:45.891 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_47 2024-09-14T09:02:45.891 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap=snap 2024-09-14T09:02:45.891 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap rm mirror/image_47@snap 2024-09-14T09:02:46.768 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing snap: 100% complete...done. 2024-09-14T09:02:46.775 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image_retry cluster2 mirror image_47 2024-09-14T09:02:46.775 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:02:46.775 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:46.775 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_47 2024-09-14T09:02:46.775 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:02:46.776 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0 2024-09-14T09:02:46.776 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_47 2024-09-14T09:02:46.776 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:02:46.776 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:46.776 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_47 2024-09-14T09:02:46.777 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_47 2024-09-14T09:02:46.834 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_47 2024-09-14T09:02:46.930 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi164.stderr:2024-09-14T09:02:46.929+0000 7f7e32524640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55b570dbe680 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-14T09:02:46.930 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi164.stderr:2024-09-14T09:02:46.929+0000 7f7e38530640 -1 rbd::mirror::ImageReplayer: 0x55b5720e2500 [3/0654f4ee-c91c-4318-964b-9a7c32deb872] operator(): start failed: (85) Interrupted system call should be restarted 2024-09-14T09:02:47.284 INFO:tasks.ceph.cluster2.osd.2.smithi164.stderr:2024-09-14T09:02:47.284+0000 7fe32301f640 -1 reset not still connected to 0x55e755137860 2024-09-14T09:02:47.285 INFO:tasks.ceph.cluster2.osd.2.smithi164.stderr:2024-09-14T09:02:47.284+0000 7fe32301f640 -1 reset not still connected to 0x55e755eb6750 2024-09-14T09:02:47.287 INFO:tasks.ceph.cluster2.osd.2.smithi164.stderr:2024-09-14T09:02:47.286+0000 7fe32301f640 -1 reset not still connected to 0x55e755eb7d40 2024-09-14T09:02:47.287 INFO:tasks.ceph.cluster2.osd.2.smithi164.stderr:2024-09-14T09:02:47.286+0000 7fe32301f640 -1 reset not still connected to 0x55e756802ea0 2024-09-14T09:02:47.287 INFO:tasks.ceph.cluster2.osd.2.smithi164.stderr:2024-09-14T09:02:47.286+0000 7fe32301f640 -1 reset not still connected to 0x55e756803930 2024-09-14T09:02:47.288 INFO:tasks.ceph.cluster2.osd.2.smithi164.stderr:2024-09-14T09:02:47.288+0000 7fe32301f640 -1 reset not still connected to 0x55e7573ad110 2024-09-14T09:02:48.102 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T09:02:48.105 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:02:48.105 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:02:48.105 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_48 2024-09-14T09:02:48.106 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_snapshot cluster2 mirror image_48 snap 2024-09-14T09:02:48.106 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:02:48.106 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:48.106 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_48 2024-09-14T09:02:48.106 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap=snap 2024-09-14T09:02:48.106 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap rm mirror/image_48@snap 2024-09-14T09:02:48.788 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing snap: 100% complete...done. 2024-09-14T09:02:48.797 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image_retry cluster2 mirror image_48 2024-09-14T09:02:48.797 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:02:48.798 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:48.798 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_48 2024-09-14T09:02:48.798 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:02:48.798 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0 2024-09-14T09:02:48.799 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_48 2024-09-14T09:02:48.799 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:02:48.799 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:48.799 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_48 2024-09-14T09:02:48.799 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_48 2024-09-14T09:02:48.855 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_48 2024-09-14T09:02:48.946 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi164.stderr:2024-09-14T09:02:48.945+0000 7f7e31d23640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55b570dbe340 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-14T09:02:48.946 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi164.stderr:2024-09-14T09:02:48.945+0000 7f7e38d31640 -1 rbd::mirror::ImageReplayer: 0x55b5720e2000 [3/d56603da-84e8-4df7-8c5e-01193a0d3227] operator(): start failed: (85) Interrupted system call should be restarted 2024-09-14T09:02:49.905 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T09:02:49.908 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:02:49.909 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:02:49.909 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_49 2024-09-14T09:02:49.909 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_snapshot cluster2 mirror image_49 snap 2024-09-14T09:02:49.909 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:02:49.909 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:49.909 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_49 2024-09-14T09:02:49.909 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap=snap 2024-09-14T09:02:49.909 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap rm mirror/image_49@snap 2024-09-14T09:02:50.608 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing snap: 100% complete...done. 2024-09-14T09:02:50.621 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image_retry cluster2 mirror image_49 2024-09-14T09:02:50.622 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:02:50.622 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:50.622 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_49 2024-09-14T09:02:50.622 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:02:50.622 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0 2024-09-14T09:02:50.622 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_49 2024-09-14T09:02:50.623 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:02:50.623 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:50.623 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_49 2024-09-14T09:02:50.623 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_49 2024-09-14T09:02:50.672 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_49 2024-09-14T09:02:50.768 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi164.stderr:2024-09-14T09:02:50.767+0000 7f3754609640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55c94be28680 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-14T09:02:50.844 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi164.stderr:2024-09-14T09:02:50.767+0000 7f375ae16640 -1 rbd::mirror::ImageReplayer: 0x55c94bf16f00 [3/e852057c-0945-4c55-b535-c147c24f9585] operator(): start failed: (85) Interrupted system call should be restarted 2024-09-14T09:02:51.719 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T09:02:51.722 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:02:51.722 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:02:51.722 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_50 2024-09-14T09:02:51.723 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_snapshot cluster2 mirror image_50 snap 2024-09-14T09:02:51.723 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:02:51.723 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:51.723 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_50 2024-09-14T09:02:51.723 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local snap=snap 2024-09-14T09:02:51.723 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap rm mirror/image_50@snap 2024-09-14T09:02:52.600 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing snap: 100% complete...done. 2024-09-14T09:02:52.611 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image_retry cluster2 mirror image_50 2024-09-14T09:02:52.611 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:02:52.611 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:52.611 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_50 2024-09-14T09:02:52.611 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:02:52.611 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0 2024-09-14T09:02:52.612 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_50 2024-09-14T09:02:52.613 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:02:52.613 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:52.613 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_50 2024-09-14T09:02:52.613 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_50 2024-09-14T09:02:52.662 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_50 2024-09-14T09:02:52.765 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi164.stderr:2024-09-14T09:02:52.764+0000 7fe44b729640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55632bea1a00 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-14T09:02:52.765 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi164.stderr:2024-09-14T09:02:52.764+0000 7fe451f36640 -1 rbd::mirror::ImageReplayer: 0x55632bf46280 [3/dc8ae139-d4cc-4aab-82f3-d300cef64dfd] operator(): start failed: (85) Interrupted system call should be restarted 2024-09-14T09:02:53.721 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T09:02:53.725 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:02:53.725 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ testlog 'TEST: image deletions should propagate' 2024-09-14T09:02:53.725 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ tee -a /tmp/tmp.IUd9LESfOA/rbd-mirror.test.log 2024-09-14T09:02:53.725 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ date '+%F %T' 2024-09-14T09:02:53.728 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ echo 2024-09-14 09:02:53 TEST: image deletions should propagate 2024-09-14T09:02:53.728 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:2024-09-14 09:02:53 TEST: image deletions should propagate 2024-09-14T09:02:53.729 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_pool_images cluster1 mirror 0 2024-09-14T09:02:53.729 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:02:53.729 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:53.729 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_count=0 2024-09-14T09:02:53.729 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T09:02:53.730 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local count 2024-09-14T09:02:53.730 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local last_count=0 2024-09-14T09:02:53.730 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ true 2024-09-14T09:02:53.730 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ seq 1 40 2024-09-14T09:02:53.730 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in `seq 1 40` 2024-09-14T09:02:53.730 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 1 -ne 1 2024-09-14T09:02:53.731 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster cluster1 -p mirror mirror pool status 2024-09-14T09:02:53.731 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ grep 'images: ' 2024-09-14T09:02:53.731 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cut '-d ' -f 2 2024-09-14T09:02:53.763 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: failed to query services: (13) Permission denied 2024-09-14T09:02:53.767 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ count=0 2024-09-14T09:02:53.767 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 0 = 0 2024-09-14T09:02:53.767 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:02:53.767 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_pool_healthy cluster1 mirror 0 2024-09-14T09:02:53.767 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:02:53.767 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:53.768 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T09:02:53.768 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state 2024-09-14T09:02:53.768 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ seq 1 40 2024-09-14T09:02:53.768 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in `seq 1 40` 2024-09-14T09:02:53.769 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 1 -ne 1 2024-09-14T09:02:53.769 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster cluster1 -p mirror mirror pool status 2024-09-14T09:02:53.769 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ grep 'image health:' 2024-09-14T09:02:53.769 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cut '-d ' -f 3 2024-09-14T09:02:53.804 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: failed to query services: (13) Permission denied 2024-09-14T09:02:53.807 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ state=OK 2024-09-14T09:02:53.808 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test OK = ERROR 2024-09-14T09:02:53.808 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test OK = OK 2024-09-14T09:02:53.808 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:02:53.808 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ seq 1 50 2024-09-14T09:02:53.809 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:02:53.809 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_1 2024-09-14T09:02:53.809 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_present cluster1 mirror image_1 deleted 2024-09-14T09:02:53.810 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:02:53.810 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:53.810 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_1 2024-09-14T09:02:53.810 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=deleted 2024-09-14T09:02:53.810 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:02:53.810 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T09:02:53.810 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:02:53.810 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_1 2024-09-14T09:02:53.810 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:02:53.810 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:02:53.810 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_1 2024-09-14T09:02:53.810 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_1 2024-09-14T09:02:53.810 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:02:53.843 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_1: (2) No such file or directory 2024-09-14T09:02:53.847 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image_id= 2024-09-14T09:02:53.847 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T09:02:53.847 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.1 2024-09-14T09:02:53.949 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_present cluster1 mirror image_1 deleted '' 2024-09-14T09:02:53.949 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:02:53.949 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:53.949 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_1 2024-09-14T09:02:53.949 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=deleted 2024-09-14T09:02:53.949 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:02:53.949 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=deleted 2024-09-14T09:02:53.950 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_image_id 2024-09-14T09:02:53.950 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_1 2024-09-14T09:02:53.950 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:02:53.950 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:02:53.950 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_1 2024-09-14T09:02:53.950 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_1 2024-09-14T09:02:53.950 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:02:53.985 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_1: (2) No such file or directory 2024-09-14T09:02:53.988 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_image_id= 2024-09-14T09:02:53.988 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:02:53.988 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test deleted = deleted 2024-09-14T09:02:53.989 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:02:53.989 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:02:53.989 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_2 2024-09-14T09:02:53.989 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_present cluster1 mirror image_2 deleted 2024-09-14T09:02:53.989 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:02:53.989 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:53.989 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_2 2024-09-14T09:02:53.989 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=deleted 2024-09-14T09:02:53.989 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:02:53.989 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T09:02:53.989 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:02:53.989 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_2 2024-09-14T09:02:53.989 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:02:53.989 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:02:53.989 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_2 2024-09-14T09:02:53.990 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_2 2024-09-14T09:02:53.990 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:02:54.023 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_2: (2) No such file or directory 2024-09-14T09:02:54.026 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image_id= 2024-09-14T09:02:54.026 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T09:02:54.026 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.1 2024-09-14T09:02:54.128 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_present cluster1 mirror image_2 deleted '' 2024-09-14T09:02:54.128 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:02:54.128 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:54.128 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_2 2024-09-14T09:02:54.128 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=deleted 2024-09-14T09:02:54.128 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:02:54.128 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=deleted 2024-09-14T09:02:54.128 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_image_id 2024-09-14T09:02:54.129 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_2 2024-09-14T09:02:54.129 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:02:54.129 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:02:54.129 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_2 2024-09-14T09:02:54.129 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_2 2024-09-14T09:02:54.129 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:02:54.163 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_2: (2) No such file or directory 2024-09-14T09:02:54.167 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_image_id= 2024-09-14T09:02:54.167 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:02:54.167 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test deleted = deleted 2024-09-14T09:02:54.167 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:02:54.167 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:02:54.167 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_3 2024-09-14T09:02:54.167 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_present cluster1 mirror image_3 deleted 2024-09-14T09:02:54.167 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:02:54.167 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:54.167 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_3 2024-09-14T09:02:54.167 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=deleted 2024-09-14T09:02:54.167 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:02:54.167 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T09:02:54.168 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:02:54.168 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_3 2024-09-14T09:02:54.168 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:02:54.168 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:02:54.168 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_3 2024-09-14T09:02:54.168 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_3 2024-09-14T09:02:54.168 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:02:54.202 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_3: (2) No such file or directory 2024-09-14T09:02:54.205 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image_id= 2024-09-14T09:02:54.205 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T09:02:54.205 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.1 2024-09-14T09:02:54.307 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_present cluster1 mirror image_3 deleted '' 2024-09-14T09:02:54.307 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:02:54.307 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:54.307 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_3 2024-09-14T09:02:54.307 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=deleted 2024-09-14T09:02:54.307 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:02:54.307 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=deleted 2024-09-14T09:02:54.307 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_image_id 2024-09-14T09:02:54.307 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_3 2024-09-14T09:02:54.307 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:02:54.307 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:02:54.308 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_3 2024-09-14T09:02:54.308 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_3 2024-09-14T09:02:54.308 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:02:54.342 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_3: (2) No such file or directory 2024-09-14T09:02:54.346 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_image_id= 2024-09-14T09:02:54.346 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:02:54.346 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test deleted = deleted 2024-09-14T09:02:54.347 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:02:54.347 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:02:54.347 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_4 2024-09-14T09:02:54.347 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_present cluster1 mirror image_4 deleted 2024-09-14T09:02:54.347 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:02:54.347 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:54.347 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_4 2024-09-14T09:02:54.347 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=deleted 2024-09-14T09:02:54.347 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:02:54.347 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T09:02:54.347 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:02:54.347 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_4 2024-09-14T09:02:54.348 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:02:54.348 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:02:54.348 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_4 2024-09-14T09:02:54.348 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_4 2024-09-14T09:02:54.348 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:02:54.380 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_4: (2) No such file or directory 2024-09-14T09:02:54.383 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image_id= 2024-09-14T09:02:54.384 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T09:02:54.384 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.1 2024-09-14T09:02:54.485 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_present cluster1 mirror image_4 deleted '' 2024-09-14T09:02:54.486 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:02:54.486 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:54.486 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_4 2024-09-14T09:02:54.486 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=deleted 2024-09-14T09:02:54.486 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:02:54.486 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=deleted 2024-09-14T09:02:54.486 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_image_id 2024-09-14T09:02:54.486 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_4 2024-09-14T09:02:54.486 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:02:54.486 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:02:54.486 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_4 2024-09-14T09:02:54.486 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_4 2024-09-14T09:02:54.486 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:02:54.522 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_4: (2) No such file or directory 2024-09-14T09:02:54.531 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_image_id= 2024-09-14T09:02:54.531 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:02:54.531 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test deleted = deleted 2024-09-14T09:02:54.531 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:02:54.531 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:02:54.532 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_5 2024-09-14T09:02:54.532 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_present cluster1 mirror image_5 deleted 2024-09-14T09:02:54.532 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:02:54.532 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:54.532 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_5 2024-09-14T09:02:54.532 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=deleted 2024-09-14T09:02:54.532 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:02:54.532 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T09:02:54.533 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:02:54.533 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_5 2024-09-14T09:02:54.533 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:02:54.533 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:02:54.533 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_5 2024-09-14T09:02:54.533 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_5 2024-09-14T09:02:54.533 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:02:54.566 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_5: (2) No such file or directory 2024-09-14T09:02:54.569 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image_id= 2024-09-14T09:02:54.569 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T09:02:54.569 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.1 2024-09-14T09:02:54.670 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_present cluster1 mirror image_5 deleted '' 2024-09-14T09:02:54.671 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:02:54.671 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:54.671 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_5 2024-09-14T09:02:54.671 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=deleted 2024-09-14T09:02:54.671 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:02:54.671 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=deleted 2024-09-14T09:02:54.671 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_image_id 2024-09-14T09:02:54.671 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_5 2024-09-14T09:02:54.671 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:02:54.671 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:02:54.671 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_5 2024-09-14T09:02:54.671 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_5 2024-09-14T09:02:54.671 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:02:54.706 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_5: (2) No such file or directory 2024-09-14T09:02:54.710 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_image_id= 2024-09-14T09:02:54.710 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:02:54.710 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test deleted = deleted 2024-09-14T09:02:54.710 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:02:54.710 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:02:54.710 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_6 2024-09-14T09:02:54.710 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_present cluster1 mirror image_6 deleted 2024-09-14T09:02:54.710 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:02:54.711 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:54.711 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_6 2024-09-14T09:02:54.711 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=deleted 2024-09-14T09:02:54.711 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:02:54.711 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T09:02:54.711 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:02:54.711 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_6 2024-09-14T09:02:54.711 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:02:54.711 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:02:54.711 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_6 2024-09-14T09:02:54.711 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_6 2024-09-14T09:02:54.711 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:02:54.745 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_6: (2) No such file or directory 2024-09-14T09:02:54.748 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image_id= 2024-09-14T09:02:54.748 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T09:02:54.749 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.1 2024-09-14T09:02:54.850 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_present cluster1 mirror image_6 deleted '' 2024-09-14T09:02:54.850 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:02:54.850 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:54.850 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_6 2024-09-14T09:02:54.850 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=deleted 2024-09-14T09:02:54.850 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:02:54.850 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=deleted 2024-09-14T09:02:54.850 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_image_id 2024-09-14T09:02:54.850 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_6 2024-09-14T09:02:54.850 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:02:54.850 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:02:54.850 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_6 2024-09-14T09:02:54.851 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_6 2024-09-14T09:02:54.851 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:02:55.087 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_6: (2) No such file or directory 2024-09-14T09:02:55.093 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_image_id= 2024-09-14T09:02:55.093 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:02:55.093 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test deleted = deleted 2024-09-14T09:02:55.093 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:02:55.093 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:02:55.093 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_7 2024-09-14T09:02:55.094 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_present cluster1 mirror image_7 deleted 2024-09-14T09:02:55.094 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:02:55.094 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:55.094 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_7 2024-09-14T09:02:55.094 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=deleted 2024-09-14T09:02:55.094 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:02:55.095 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T09:02:55.095 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:02:55.095 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_7 2024-09-14T09:02:55.095 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:02:55.095 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:02:55.096 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_7 2024-09-14T09:02:55.096 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_7 2024-09-14T09:02:55.096 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:02:55.126 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_7: (2) No such file or directory 2024-09-14T09:02:55.129 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image_id= 2024-09-14T09:02:55.129 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T09:02:55.129 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.1 2024-09-14T09:02:55.230 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_present cluster1 mirror image_7 deleted '' 2024-09-14T09:02:55.231 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:02:55.231 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:55.231 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_7 2024-09-14T09:02:55.231 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=deleted 2024-09-14T09:02:55.231 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:02:55.231 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=deleted 2024-09-14T09:02:55.231 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_image_id 2024-09-14T09:02:55.231 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_7 2024-09-14T09:02:55.231 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:02:55.231 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:02:55.231 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_7 2024-09-14T09:02:55.231 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_7 2024-09-14T09:02:55.232 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:02:55.265 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_7: (2) No such file or directory 2024-09-14T09:02:55.269 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_image_id= 2024-09-14T09:02:55.269 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:02:55.269 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test deleted = deleted 2024-09-14T09:02:55.269 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:02:55.269 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:02:55.270 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_8 2024-09-14T09:02:55.270 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_present cluster1 mirror image_8 deleted 2024-09-14T09:02:55.270 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:02:55.270 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:55.270 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_8 2024-09-14T09:02:55.270 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=deleted 2024-09-14T09:02:55.270 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:02:55.270 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T09:02:55.270 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:02:55.271 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_8 2024-09-14T09:02:55.271 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:02:55.271 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:02:55.271 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_8 2024-09-14T09:02:55.271 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_8 2024-09-14T09:02:55.271 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:02:55.505 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_8: (2) No such file or directory 2024-09-14T09:02:55.509 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image_id= 2024-09-14T09:02:55.509 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T09:02:55.509 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.1 2024-09-14T09:02:55.611 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_present cluster1 mirror image_8 deleted '' 2024-09-14T09:02:55.611 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:02:55.611 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:55.611 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_8 2024-09-14T09:02:55.611 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=deleted 2024-09-14T09:02:55.611 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:02:55.611 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=deleted 2024-09-14T09:02:55.611 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_image_id 2024-09-14T09:02:55.611 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_8 2024-09-14T09:02:55.611 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:02:55.612 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:02:55.612 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_8 2024-09-14T09:02:55.612 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_8 2024-09-14T09:02:55.612 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:02:55.647 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_8: (2) No such file or directory 2024-09-14T09:02:55.651 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_image_id= 2024-09-14T09:02:55.651 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:02:55.651 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test deleted = deleted 2024-09-14T09:02:55.651 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:02:55.651 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:02:55.651 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_9 2024-09-14T09:02:55.651 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_present cluster1 mirror image_9 deleted 2024-09-14T09:02:55.652 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:02:55.652 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:55.652 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_9 2024-09-14T09:02:55.652 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=deleted 2024-09-14T09:02:55.652 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:02:55.652 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T09:02:55.652 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:02:55.652 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_9 2024-09-14T09:02:55.652 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:02:55.652 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:02:55.652 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_9 2024-09-14T09:02:55.652 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_9 2024-09-14T09:02:55.652 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:02:55.685 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_9: (2) No such file or directory 2024-09-14T09:02:55.688 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image_id= 2024-09-14T09:02:55.688 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T09:02:55.689 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.1 2024-09-14T09:02:55.791 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_present cluster1 mirror image_9 deleted '' 2024-09-14T09:02:55.791 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:02:55.791 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:55.791 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_9 2024-09-14T09:02:55.792 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=deleted 2024-09-14T09:02:55.792 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:02:55.792 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=deleted 2024-09-14T09:02:55.792 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_image_id 2024-09-14T09:02:55.792 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_9 2024-09-14T09:02:55.792 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:02:55.792 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:02:55.792 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_9 2024-09-14T09:02:55.792 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_9 2024-09-14T09:02:55.792 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:02:55.826 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_9: (2) No such file or directory 2024-09-14T09:02:55.830 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_image_id= 2024-09-14T09:02:55.830 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:02:55.830 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test deleted = deleted 2024-09-14T09:02:55.830 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:02:55.830 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:02:55.830 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_10 2024-09-14T09:02:55.830 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_present cluster1 mirror image_10 deleted 2024-09-14T09:02:55.831 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:02:55.831 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:55.831 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_10 2024-09-14T09:02:55.831 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=deleted 2024-09-14T09:02:55.831 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:02:55.831 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T09:02:55.831 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:02:55.831 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_10 2024-09-14T09:02:55.831 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:02:55.831 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:02:55.831 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_10 2024-09-14T09:02:55.831 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_10 2024-09-14T09:02:55.831 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:02:55.864 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_10: (2) No such file or directory 2024-09-14T09:02:55.867 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image_id= 2024-09-14T09:02:55.867 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T09:02:55.867 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.1 2024-09-14T09:02:55.969 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_present cluster1 mirror image_10 deleted '' 2024-09-14T09:02:55.969 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:02:55.969 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:55.969 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_10 2024-09-14T09:02:55.969 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=deleted 2024-09-14T09:02:55.970 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:02:55.970 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=deleted 2024-09-14T09:02:55.970 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_image_id 2024-09-14T09:02:55.970 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_10 2024-09-14T09:02:55.970 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:02:55.970 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:02:55.970 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_10 2024-09-14T09:02:55.970 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_10 2024-09-14T09:02:55.970 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:02:56.003 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_10: (2) No such file or directory 2024-09-14T09:02:56.008 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_image_id= 2024-09-14T09:02:56.008 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:02:56.008 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test deleted = deleted 2024-09-14T09:02:56.008 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:02:56.008 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:02:56.009 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_11 2024-09-14T09:02:56.009 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_present cluster1 mirror image_11 deleted 2024-09-14T09:02:56.009 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:02:56.009 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:56.009 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_11 2024-09-14T09:02:56.009 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=deleted 2024-09-14T09:02:56.009 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:02:56.009 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T09:02:56.009 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:02:56.009 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_11 2024-09-14T09:02:56.009 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:02:56.009 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:02:56.009 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_11 2024-09-14T09:02:56.009 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_11 2024-09-14T09:02:56.010 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:02:56.043 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_11: (2) No such file or directory 2024-09-14T09:02:56.046 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image_id= 2024-09-14T09:02:56.046 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T09:02:56.046 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.1 2024-09-14T09:02:56.147 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_present cluster1 mirror image_11 deleted '' 2024-09-14T09:02:56.147 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:02:56.147 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:56.147 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_11 2024-09-14T09:02:56.148 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=deleted 2024-09-14T09:02:56.148 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:02:56.148 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=deleted 2024-09-14T09:02:56.148 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_image_id 2024-09-14T09:02:56.148 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_11 2024-09-14T09:02:56.148 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:02:56.148 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:02:56.148 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_11 2024-09-14T09:02:56.148 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_11 2024-09-14T09:02:56.148 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:02:56.182 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_11: (2) No such file or directory 2024-09-14T09:02:56.186 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_image_id= 2024-09-14T09:02:56.186 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:02:56.186 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test deleted = deleted 2024-09-14T09:02:56.187 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:02:56.187 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:02:56.187 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_12 2024-09-14T09:02:56.187 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_present cluster1 mirror image_12 deleted 2024-09-14T09:02:56.187 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:02:56.187 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:56.187 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_12 2024-09-14T09:02:56.187 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=deleted 2024-09-14T09:02:56.187 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:02:56.187 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T09:02:56.187 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:02:56.187 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_12 2024-09-14T09:02:56.187 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:02:56.187 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:02:56.188 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_12 2024-09-14T09:02:56.188 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_12 2024-09-14T09:02:56.188 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:02:56.221 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_12: (2) No such file or directory 2024-09-14T09:02:56.224 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image_id= 2024-09-14T09:02:56.224 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T09:02:56.224 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.1 2024-09-14T09:02:56.326 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_present cluster1 mirror image_12 deleted '' 2024-09-14T09:02:56.326 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:02:56.326 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:56.326 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_12 2024-09-14T09:02:56.326 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=deleted 2024-09-14T09:02:56.326 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:02:56.326 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=deleted 2024-09-14T09:02:56.326 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_image_id 2024-09-14T09:02:56.327 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_12 2024-09-14T09:02:56.327 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:02:56.327 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:02:56.327 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_12 2024-09-14T09:02:56.327 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_12 2024-09-14T09:02:56.327 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:02:56.366 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_12: (2) No such file or directory 2024-09-14T09:02:56.366 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_image_id= 2024-09-14T09:02:56.366 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:02:56.366 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test deleted = deleted 2024-09-14T09:02:56.366 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:02:56.366 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:02:56.366 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_13 2024-09-14T09:02:56.366 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_present cluster1 mirror image_13 deleted 2024-09-14T09:02:56.366 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:02:56.366 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:56.367 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_13 2024-09-14T09:02:56.367 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=deleted 2024-09-14T09:02:56.367 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:02:56.367 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T09:02:56.367 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:02:56.367 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_13 2024-09-14T09:02:56.367 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:02:56.367 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:02:56.367 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_13 2024-09-14T09:02:56.367 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_13 2024-09-14T09:02:56.367 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:02:56.400 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_13: (2) No such file or directory 2024-09-14T09:02:56.403 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image_id= 2024-09-14T09:02:56.403 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T09:02:56.403 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.1 2024-09-14T09:02:56.505 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_present cluster1 mirror image_13 deleted '' 2024-09-14T09:02:56.505 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:02:56.505 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:56.505 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_13 2024-09-14T09:02:56.506 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=deleted 2024-09-14T09:02:56.506 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:02:56.506 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=deleted 2024-09-14T09:02:56.506 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_image_id 2024-09-14T09:02:56.506 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_13 2024-09-14T09:02:56.506 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:02:56.506 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:02:56.506 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_13 2024-09-14T09:02:56.506 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_13 2024-09-14T09:02:56.506 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:02:56.541 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_13: (2) No such file or directory 2024-09-14T09:02:56.546 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_image_id= 2024-09-14T09:02:56.546 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:02:56.546 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test deleted = deleted 2024-09-14T09:02:56.546 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:02:56.546 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:02:56.546 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_14 2024-09-14T09:02:56.546 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_present cluster1 mirror image_14 deleted 2024-09-14T09:02:56.547 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:02:56.547 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:56.547 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_14 2024-09-14T09:02:56.547 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=deleted 2024-09-14T09:02:56.547 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:02:56.547 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T09:02:56.547 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:02:56.547 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_14 2024-09-14T09:02:56.547 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:02:56.547 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:02:56.547 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_14 2024-09-14T09:02:56.547 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_14 2024-09-14T09:02:56.547 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:02:56.580 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_14: (2) No such file or directory 2024-09-14T09:02:56.583 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image_id= 2024-09-14T09:02:56.584 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T09:02:56.584 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.1 2024-09-14T09:02:56.686 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_present cluster1 mirror image_14 deleted '' 2024-09-14T09:02:56.686 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:02:56.686 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:56.686 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_14 2024-09-14T09:02:56.686 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=deleted 2024-09-14T09:02:56.686 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:02:56.686 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=deleted 2024-09-14T09:02:56.686 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_image_id 2024-09-14T09:02:56.686 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_14 2024-09-14T09:02:56.686 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:02:56.686 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:02:56.687 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_14 2024-09-14T09:02:56.687 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_14 2024-09-14T09:02:56.687 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:02:56.722 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_14: (2) No such file or directory 2024-09-14T09:02:56.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_image_id= 2024-09-14T09:02:56.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:02:56.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test deleted = deleted 2024-09-14T09:02:56.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:02:56.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:02:56.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_15 2024-09-14T09:02:56.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_present cluster1 mirror image_15 deleted 2024-09-14T09:02:56.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:02:56.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:56.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_15 2024-09-14T09:02:56.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=deleted 2024-09-14T09:02:56.727 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:02:56.727 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T09:02:56.727 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:02:56.727 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_15 2024-09-14T09:02:56.727 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:02:56.727 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:02:56.727 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_15 2024-09-14T09:02:56.727 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_15 2024-09-14T09:02:56.727 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:02:56.760 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_15: (2) No such file or directory 2024-09-14T09:02:56.763 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image_id= 2024-09-14T09:02:56.763 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T09:02:56.763 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.1 2024-09-14T09:02:56.865 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_present cluster1 mirror image_15 deleted '' 2024-09-14T09:02:56.865 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:02:56.865 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:56.866 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_15 2024-09-14T09:02:56.866 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=deleted 2024-09-14T09:02:56.866 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:02:56.866 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=deleted 2024-09-14T09:02:56.866 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_image_id 2024-09-14T09:02:56.866 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_15 2024-09-14T09:02:56.866 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:02:56.866 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:02:56.866 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_15 2024-09-14T09:02:56.866 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_15 2024-09-14T09:02:56.866 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:02:56.900 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_15: (2) No such file or directory 2024-09-14T09:02:56.904 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_image_id= 2024-09-14T09:02:56.904 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:02:56.904 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test deleted = deleted 2024-09-14T09:02:56.904 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:02:56.905 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:02:56.905 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_16 2024-09-14T09:02:56.905 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_present cluster1 mirror image_16 deleted 2024-09-14T09:02:56.905 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:02:56.905 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:56.905 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_16 2024-09-14T09:02:56.905 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=deleted 2024-09-14T09:02:56.905 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:02:56.906 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T09:02:56.906 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:02:56.906 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_16 2024-09-14T09:02:56.906 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:02:56.906 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:02:56.906 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_16 2024-09-14T09:02:56.906 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_16 2024-09-14T09:02:56.906 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:02:56.938 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_16: (2) No such file or directory 2024-09-14T09:02:56.941 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image_id= 2024-09-14T09:02:56.941 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T09:02:56.941 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.1 2024-09-14T09:02:57.043 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_present cluster1 mirror image_16 deleted '' 2024-09-14T09:02:57.043 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:02:57.043 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:57.043 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_16 2024-09-14T09:02:57.043 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=deleted 2024-09-14T09:02:57.044 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:02:57.044 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=deleted 2024-09-14T09:02:57.044 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_image_id 2024-09-14T09:02:57.044 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_16 2024-09-14T09:02:57.044 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:02:57.044 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:02:57.044 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_16 2024-09-14T09:02:57.044 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_16 2024-09-14T09:02:57.044 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:02:57.077 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_16: (2) No such file or directory 2024-09-14T09:02:57.081 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_image_id= 2024-09-14T09:02:57.081 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:02:57.081 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test deleted = deleted 2024-09-14T09:02:57.081 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:02:57.081 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:02:57.081 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_17 2024-09-14T09:02:57.082 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_present cluster1 mirror image_17 deleted 2024-09-14T09:02:57.082 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:02:57.082 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:57.082 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_17 2024-09-14T09:02:57.082 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=deleted 2024-09-14T09:02:57.082 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:02:57.082 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T09:02:57.082 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:02:57.082 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_17 2024-09-14T09:02:57.082 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:02:57.082 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:02:57.082 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_17 2024-09-14T09:02:57.082 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_17 2024-09-14T09:02:57.082 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:02:57.115 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_17: (2) No such file or directory 2024-09-14T09:02:57.118 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image_id= 2024-09-14T09:02:57.118 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T09:02:57.118 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.1 2024-09-14T09:02:57.220 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_present cluster1 mirror image_17 deleted '' 2024-09-14T09:02:57.220 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:02:57.220 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:57.221 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_17 2024-09-14T09:02:57.221 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=deleted 2024-09-14T09:02:57.221 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:02:57.221 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=deleted 2024-09-14T09:02:57.221 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_image_id 2024-09-14T09:02:57.221 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_17 2024-09-14T09:02:57.221 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:02:57.221 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:02:57.221 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_17 2024-09-14T09:02:57.221 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_17 2024-09-14T09:02:57.221 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:02:57.255 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_17: (2) No such file or directory 2024-09-14T09:02:57.259 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_image_id= 2024-09-14T09:02:57.259 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:02:57.259 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test deleted = deleted 2024-09-14T09:02:57.259 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:02:57.259 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:02:57.259 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_18 2024-09-14T09:02:57.259 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_present cluster1 mirror image_18 deleted 2024-09-14T09:02:57.259 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:02:57.259 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:57.259 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_18 2024-09-14T09:02:57.260 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=deleted 2024-09-14T09:02:57.260 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:02:57.260 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T09:02:57.260 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:02:57.260 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_18 2024-09-14T09:02:57.260 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:02:57.260 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:02:57.260 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_18 2024-09-14T09:02:57.260 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_18 2024-09-14T09:02:57.260 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:02:57.496 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_18: (2) No such file or directory 2024-09-14T09:02:57.499 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image_id= 2024-09-14T09:02:57.499 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T09:02:57.499 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.1 2024-09-14T09:02:57.601 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_present cluster1 mirror image_18 deleted '' 2024-09-14T09:02:57.601 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:02:57.601 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:57.601 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_18 2024-09-14T09:02:57.601 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=deleted 2024-09-14T09:02:57.601 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:02:57.601 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=deleted 2024-09-14T09:02:57.601 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_image_id 2024-09-14T09:02:57.602 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_18 2024-09-14T09:02:57.602 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:02:57.602 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:02:57.602 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_18 2024-09-14T09:02:57.602 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_18 2024-09-14T09:02:57.602 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:02:57.640 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_18: (2) No such file or directory 2024-09-14T09:02:57.645 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_image_id= 2024-09-14T09:02:57.645 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:02:57.645 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test deleted = deleted 2024-09-14T09:02:57.645 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:02:57.645 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:02:57.645 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_19 2024-09-14T09:02:57.645 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_present cluster1 mirror image_19 deleted 2024-09-14T09:02:57.645 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:02:57.645 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:57.645 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_19 2024-09-14T09:02:57.645 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=deleted 2024-09-14T09:02:57.645 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:02:57.646 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T09:02:57.646 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:02:57.646 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_19 2024-09-14T09:02:57.646 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:02:57.646 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:02:57.646 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_19 2024-09-14T09:02:57.646 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_19 2024-09-14T09:02:57.646 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:02:57.680 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_19: (2) No such file or directory 2024-09-14T09:02:57.682 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image_id= 2024-09-14T09:02:57.682 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T09:02:57.683 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.1 2024-09-14T09:02:57.785 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_present cluster1 mirror image_19 deleted '' 2024-09-14T09:02:57.785 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:02:57.785 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:57.785 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_19 2024-09-14T09:02:57.785 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=deleted 2024-09-14T09:02:57.785 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:02:57.785 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=deleted 2024-09-14T09:02:57.785 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_image_id 2024-09-14T09:02:57.785 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_19 2024-09-14T09:02:57.785 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:02:57.785 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:02:57.785 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_19 2024-09-14T09:02:57.786 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_19 2024-09-14T09:02:57.786 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:02:57.820 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_19: (2) No such file or directory 2024-09-14T09:02:57.824 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_image_id= 2024-09-14T09:02:57.824 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:02:57.824 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test deleted = deleted 2024-09-14T09:02:57.824 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:02:57.824 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:02:57.825 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_20 2024-09-14T09:02:57.825 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_present cluster1 mirror image_20 deleted 2024-09-14T09:02:57.825 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:02:57.825 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:57.825 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_20 2024-09-14T09:02:57.825 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=deleted 2024-09-14T09:02:57.825 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:02:57.825 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T09:02:57.825 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:02:57.825 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_20 2024-09-14T09:02:57.825 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:02:57.825 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:02:57.825 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_20 2024-09-14T09:02:57.825 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_20 2024-09-14T09:02:57.826 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:02:57.858 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_20: (2) No such file or directory 2024-09-14T09:02:57.861 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image_id= 2024-09-14T09:02:57.862 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T09:02:57.862 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.1 2024-09-14T09:02:57.964 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_present cluster1 mirror image_20 deleted '' 2024-09-14T09:02:57.964 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:02:57.964 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:57.965 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_20 2024-09-14T09:02:57.965 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=deleted 2024-09-14T09:02:57.965 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:02:57.965 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=deleted 2024-09-14T09:02:57.965 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_image_id 2024-09-14T09:02:57.965 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_20 2024-09-14T09:02:57.965 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:02:57.965 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:02:57.965 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_20 2024-09-14T09:02:57.965 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_20 2024-09-14T09:02:57.965 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:02:58.000 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_20: (2) No such file or directory 2024-09-14T09:02:58.002 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_image_id= 2024-09-14T09:02:58.002 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:02:58.003 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test deleted = deleted 2024-09-14T09:02:58.003 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:02:58.003 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:02:58.003 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_21 2024-09-14T09:02:58.003 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_present cluster1 mirror image_21 deleted 2024-09-14T09:02:58.003 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:02:58.003 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:58.003 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_21 2024-09-14T09:02:58.003 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=deleted 2024-09-14T09:02:58.003 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:02:58.003 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T09:02:58.003 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:02:58.003 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_21 2024-09-14T09:02:58.003 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:02:58.004 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:02:58.004 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_21 2024-09-14T09:02:58.004 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_21 2024-09-14T09:02:58.004 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:02:58.036 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_21: (2) No such file or directory 2024-09-14T09:02:58.039 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image_id= 2024-09-14T09:02:58.040 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T09:02:58.040 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.1 2024-09-14T09:02:58.141 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_present cluster1 mirror image_21 deleted '' 2024-09-14T09:02:58.141 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:02:58.141 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:58.142 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_21 2024-09-14T09:02:58.142 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=deleted 2024-09-14T09:02:58.142 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:02:58.142 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=deleted 2024-09-14T09:02:58.142 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_image_id 2024-09-14T09:02:58.142 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_21 2024-09-14T09:02:58.142 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:02:58.142 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:02:58.142 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_21 2024-09-14T09:02:58.142 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_21 2024-09-14T09:02:58.142 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:02:58.176 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_21: (2) No such file or directory 2024-09-14T09:02:58.181 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_image_id= 2024-09-14T09:02:58.181 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:02:58.181 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test deleted = deleted 2024-09-14T09:02:58.181 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:02:58.181 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:02:58.181 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_22 2024-09-14T09:02:58.181 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_present cluster1 mirror image_22 deleted 2024-09-14T09:02:58.181 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:02:58.181 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:58.181 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_22 2024-09-14T09:02:58.181 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=deleted 2024-09-14T09:02:58.181 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:02:58.181 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T09:02:58.182 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:02:58.182 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_22 2024-09-14T09:02:58.182 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:02:58.182 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:02:58.182 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_22 2024-09-14T09:02:58.182 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_22 2024-09-14T09:02:58.182 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:02:58.216 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_22: (2) No such file or directory 2024-09-14T09:02:58.219 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image_id= 2024-09-14T09:02:58.219 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T09:02:58.219 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.1 2024-09-14T09:02:58.321 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_present cluster1 mirror image_22 deleted '' 2024-09-14T09:02:58.321 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:02:58.321 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:58.321 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_22 2024-09-14T09:02:58.321 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=deleted 2024-09-14T09:02:58.321 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:02:58.321 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=deleted 2024-09-14T09:02:58.321 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_image_id 2024-09-14T09:02:58.322 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_22 2024-09-14T09:02:58.322 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:02:58.322 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:02:58.322 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_22 2024-09-14T09:02:58.322 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_22 2024-09-14T09:02:58.322 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:02:58.356 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_22: (2) No such file or directory 2024-09-14T09:02:58.361 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_image_id= 2024-09-14T09:02:58.361 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:02:58.361 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test deleted = deleted 2024-09-14T09:02:58.361 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:02:58.361 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:02:58.361 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_23 2024-09-14T09:02:58.361 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_present cluster1 mirror image_23 deleted 2024-09-14T09:02:58.361 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:02:58.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:58.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_23 2024-09-14T09:02:58.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=deleted 2024-09-14T09:02:58.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:02:58.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T09:02:58.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:02:58.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_23 2024-09-14T09:02:58.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:02:58.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:02:58.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_23 2024-09-14T09:02:58.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_23 2024-09-14T09:02:58.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:02:58.396 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_23: (2) No such file or directory 2024-09-14T09:02:58.399 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image_id= 2024-09-14T09:02:58.399 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T09:02:58.399 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.1 2024-09-14T09:02:58.501 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_present cluster1 mirror image_23 deleted '' 2024-09-14T09:02:58.501 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:02:58.502 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:58.502 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_23 2024-09-14T09:02:58.502 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=deleted 2024-09-14T09:02:58.502 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:02:58.502 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=deleted 2024-09-14T09:02:58.502 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_image_id 2024-09-14T09:02:58.502 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_23 2024-09-14T09:02:58.502 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:02:58.502 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:02:58.502 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_23 2024-09-14T09:02:58.503 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_23 2024-09-14T09:02:58.503 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:02:58.540 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_23: (2) No such file or directory 2024-09-14T09:02:58.545 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_image_id= 2024-09-14T09:02:58.545 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:02:58.545 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test deleted = deleted 2024-09-14T09:02:58.545 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:02:58.545 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:02:58.545 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_24 2024-09-14T09:02:58.545 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_present cluster1 mirror image_24 deleted 2024-09-14T09:02:58.545 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:02:58.545 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:58.546 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_24 2024-09-14T09:02:58.546 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=deleted 2024-09-14T09:02:58.546 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:02:58.546 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T09:02:58.546 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:02:58.546 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_24 2024-09-14T09:02:58.546 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:02:58.546 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:02:58.546 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_24 2024-09-14T09:02:58.546 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_24 2024-09-14T09:02:58.546 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:02:58.580 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_24: (2) No such file or directory 2024-09-14T09:02:58.582 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image_id= 2024-09-14T09:02:58.583 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T09:02:58.583 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.1 2024-09-14T09:02:58.685 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_present cluster1 mirror image_24 deleted '' 2024-09-14T09:02:58.685 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:02:58.685 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:58.685 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_24 2024-09-14T09:02:58.685 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=deleted 2024-09-14T09:02:58.685 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:02:58.685 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=deleted 2024-09-14T09:02:58.685 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_image_id 2024-09-14T09:02:58.685 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_24 2024-09-14T09:02:58.685 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:02:58.685 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:02:58.685 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_24 2024-09-14T09:02:58.686 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_24 2024-09-14T09:02:58.686 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:02:58.721 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_24: (2) No such file or directory 2024-09-14T09:02:58.725 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_image_id= 2024-09-14T09:02:58.725 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:02:58.725 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test deleted = deleted 2024-09-14T09:02:58.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:02:58.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:02:58.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_25 2024-09-14T09:02:58.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_present cluster1 mirror image_25 deleted 2024-09-14T09:02:58.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:02:58.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:58.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_25 2024-09-14T09:02:58.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=deleted 2024-09-14T09:02:58.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:02:58.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T09:02:58.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:02:58.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_25 2024-09-14T09:02:58.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:02:58.727 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:02:58.727 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_25 2024-09-14T09:02:58.727 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_25 2024-09-14T09:02:58.727 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:02:58.760 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_25: (2) No such file or directory 2024-09-14T09:02:58.763 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image_id= 2024-09-14T09:02:58.764 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T09:02:58.764 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.1 2024-09-14T09:02:58.865 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_present cluster1 mirror image_25 deleted '' 2024-09-14T09:02:58.866 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:02:58.866 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:58.866 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_25 2024-09-14T09:02:58.866 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=deleted 2024-09-14T09:02:58.866 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:02:58.866 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=deleted 2024-09-14T09:02:58.866 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_image_id 2024-09-14T09:02:58.866 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_25 2024-09-14T09:02:58.866 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:02:58.866 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:02:58.866 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_25 2024-09-14T09:02:58.867 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_25 2024-09-14T09:02:58.867 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:02:58.901 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_25: (2) No such file or directory 2024-09-14T09:02:58.905 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_image_id= 2024-09-14T09:02:58.905 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:02:58.905 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test deleted = deleted 2024-09-14T09:02:58.905 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:02:58.905 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:02:58.905 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_26 2024-09-14T09:02:58.906 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_present cluster1 mirror image_26 deleted 2024-09-14T09:02:58.906 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:02:58.906 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:58.906 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_26 2024-09-14T09:02:58.906 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=deleted 2024-09-14T09:02:58.906 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:02:58.906 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T09:02:58.906 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:02:58.906 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_26 2024-09-14T09:02:58.906 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:02:58.906 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:02:58.906 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_26 2024-09-14T09:02:58.906 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_26 2024-09-14T09:02:58.907 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:02:58.939 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_26: (2) No such file or directory 2024-09-14T09:02:58.942 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image_id= 2024-09-14T09:02:58.943 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T09:02:58.943 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.1 2024-09-14T09:02:59.045 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_present cluster1 mirror image_26 deleted '' 2024-09-14T09:02:59.045 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:02:59.045 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:59.045 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_26 2024-09-14T09:02:59.045 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=deleted 2024-09-14T09:02:59.045 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:02:59.045 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=deleted 2024-09-14T09:02:59.045 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_image_id 2024-09-14T09:02:59.045 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_26 2024-09-14T09:02:59.045 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:02:59.045 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:02:59.046 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_26 2024-09-14T09:02:59.046 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_26 2024-09-14T09:02:59.046 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:02:59.080 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_26: (2) No such file or directory 2024-09-14T09:02:59.086 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_image_id= 2024-09-14T09:02:59.086 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:02:59.086 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test deleted = deleted 2024-09-14T09:02:59.086 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:02:59.086 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:02:59.086 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_27 2024-09-14T09:02:59.086 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_present cluster1 mirror image_27 deleted 2024-09-14T09:02:59.086 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:02:59.086 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:59.086 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_27 2024-09-14T09:02:59.086 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=deleted 2024-09-14T09:02:59.086 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:02:59.087 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T09:02:59.087 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:02:59.087 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_27 2024-09-14T09:02:59.087 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:02:59.087 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:02:59.087 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_27 2024-09-14T09:02:59.087 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_27 2024-09-14T09:02:59.087 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:02:59.120 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_27: (2) No such file or directory 2024-09-14T09:02:59.123 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image_id= 2024-09-14T09:02:59.123 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T09:02:59.123 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.1 2024-09-14T09:02:59.225 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_present cluster1 mirror image_27 deleted '' 2024-09-14T09:02:59.225 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:02:59.225 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:59.225 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_27 2024-09-14T09:02:59.225 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=deleted 2024-09-14T09:02:59.225 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:02:59.225 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=deleted 2024-09-14T09:02:59.225 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_image_id 2024-09-14T09:02:59.225 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_27 2024-09-14T09:02:59.226 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:02:59.226 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:02:59.226 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_27 2024-09-14T09:02:59.226 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_27 2024-09-14T09:02:59.226 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:02:59.260 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_27: (2) No such file or directory 2024-09-14T09:02:59.264 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_image_id= 2024-09-14T09:02:59.264 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:02:59.264 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test deleted = deleted 2024-09-14T09:02:59.264 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:02:59.265 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:02:59.265 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_28 2024-09-14T09:02:59.265 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_present cluster1 mirror image_28 deleted 2024-09-14T09:02:59.265 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:02:59.265 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:59.265 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_28 2024-09-14T09:02:59.265 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=deleted 2024-09-14T09:02:59.265 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:02:59.265 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T09:02:59.265 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:02:59.265 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_28 2024-09-14T09:02:59.265 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:02:59.265 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:02:59.265 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_28 2024-09-14T09:02:59.266 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_28 2024-09-14T09:02:59.266 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:02:59.299 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_28: (2) No such file or directory 2024-09-14T09:02:59.302 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image_id= 2024-09-14T09:02:59.302 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T09:02:59.302 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.1 2024-09-14T09:02:59.404 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_present cluster1 mirror image_28 deleted '' 2024-09-14T09:02:59.404 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:02:59.404 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:59.404 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_28 2024-09-14T09:02:59.405 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=deleted 2024-09-14T09:02:59.405 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:02:59.405 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=deleted 2024-09-14T09:02:59.405 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_image_id 2024-09-14T09:02:59.405 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_28 2024-09-14T09:02:59.405 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:02:59.405 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:02:59.405 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_28 2024-09-14T09:02:59.405 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_28 2024-09-14T09:02:59.405 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:02:59.441 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_28: (2) No such file or directory 2024-09-14T09:02:59.445 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_image_id= 2024-09-14T09:02:59.446 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:02:59.446 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test deleted = deleted 2024-09-14T09:02:59.446 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:02:59.446 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:02:59.446 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_29 2024-09-14T09:02:59.446 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_present cluster1 mirror image_29 deleted 2024-09-14T09:02:59.446 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:02:59.446 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:59.446 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_29 2024-09-14T09:02:59.446 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=deleted 2024-09-14T09:02:59.446 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:02:59.446 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T09:02:59.446 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:02:59.447 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_29 2024-09-14T09:02:59.447 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:02:59.447 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:02:59.447 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_29 2024-09-14T09:02:59.447 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_29 2024-09-14T09:02:59.447 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:02:59.480 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_29: (2) No such file or directory 2024-09-14T09:02:59.483 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image_id= 2024-09-14T09:02:59.483 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T09:02:59.483 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.1 2024-09-14T09:02:59.585 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_present cluster1 mirror image_29 deleted '' 2024-09-14T09:02:59.585 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:02:59.585 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:59.585 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_29 2024-09-14T09:02:59.585 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=deleted 2024-09-14T09:02:59.585 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:02:59.586 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=deleted 2024-09-14T09:02:59.586 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_image_id 2024-09-14T09:02:59.586 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_29 2024-09-14T09:02:59.586 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:02:59.586 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:02:59.586 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_29 2024-09-14T09:02:59.586 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_29 2024-09-14T09:02:59.586 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:02:59.624 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_29: (2) No such file or directory 2024-09-14T09:02:59.629 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_image_id= 2024-09-14T09:02:59.629 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:02:59.629 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test deleted = deleted 2024-09-14T09:02:59.629 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:02:59.629 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:02:59.629 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_30 2024-09-14T09:02:59.629 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_present cluster1 mirror image_30 deleted 2024-09-14T09:02:59.629 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:02:59.629 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:59.629 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_30 2024-09-14T09:02:59.629 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=deleted 2024-09-14T09:02:59.629 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:02:59.629 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T09:02:59.630 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:02:59.630 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_30 2024-09-14T09:02:59.630 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:02:59.630 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:02:59.630 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_30 2024-09-14T09:02:59.630 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_30 2024-09-14T09:02:59.630 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:02:59.664 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_30: (2) No such file or directory 2024-09-14T09:02:59.667 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image_id= 2024-09-14T09:02:59.667 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T09:02:59.667 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.1 2024-09-14T09:02:59.769 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_present cluster1 mirror image_30 deleted '' 2024-09-14T09:02:59.769 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:02:59.769 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:59.769 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_30 2024-09-14T09:02:59.769 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=deleted 2024-09-14T09:02:59.769 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:02:59.769 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=deleted 2024-09-14T09:02:59.769 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_image_id 2024-09-14T09:02:59.769 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_30 2024-09-14T09:02:59.769 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:02:59.770 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:02:59.770 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_30 2024-09-14T09:02:59.770 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_30 2024-09-14T09:02:59.770 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:02:59.804 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_30: (2) No such file or directory 2024-09-14T09:02:59.808 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_image_id= 2024-09-14T09:02:59.809 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:02:59.809 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test deleted = deleted 2024-09-14T09:02:59.809 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:02:59.810 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:02:59.810 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_31 2024-09-14T09:02:59.810 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_present cluster1 mirror image_31 deleted 2024-09-14T09:02:59.810 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:02:59.810 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:59.810 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_31 2024-09-14T09:02:59.810 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=deleted 2024-09-14T09:02:59.810 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:02:59.810 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T09:02:59.810 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:02:59.811 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_31 2024-09-14T09:02:59.811 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:02:59.811 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:02:59.811 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_31 2024-09-14T09:02:59.811 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_31 2024-09-14T09:02:59.811 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:02:59.844 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_31: (2) No such file or directory 2024-09-14T09:02:59.846 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image_id= 2024-09-14T09:02:59.846 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T09:02:59.846 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.1 2024-09-14T09:02:59.948 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_present cluster1 mirror image_31 deleted '' 2024-09-14T09:02:59.948 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:02:59.948 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:59.948 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_31 2024-09-14T09:02:59.948 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=deleted 2024-09-14T09:02:59.949 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:02:59.949 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=deleted 2024-09-14T09:02:59.949 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_image_id 2024-09-14T09:02:59.949 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_31 2024-09-14T09:02:59.949 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:02:59.949 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:02:59.949 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_31 2024-09-14T09:02:59.949 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_31 2024-09-14T09:02:59.949 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:02:59.983 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_31: (2) No such file or directory 2024-09-14T09:02:59.987 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_image_id= 2024-09-14T09:02:59.988 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:02:59.988 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test deleted = deleted 2024-09-14T09:02:59.988 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:02:59.988 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:02:59.988 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_32 2024-09-14T09:02:59.988 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_present cluster1 mirror image_32 deleted 2024-09-14T09:02:59.988 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:02:59.988 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:02:59.988 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_32 2024-09-14T09:02:59.988 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=deleted 2024-09-14T09:02:59.988 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:02:59.988 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T09:02:59.988 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:02:59.989 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_32 2024-09-14T09:02:59.989 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:02:59.989 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:02:59.989 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_32 2024-09-14T09:02:59.989 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_32 2024-09-14T09:02:59.989 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:03:00.022 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_32: (2) No such file or directory 2024-09-14T09:03:00.026 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image_id= 2024-09-14T09:03:00.026 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T09:03:00.026 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.1 2024-09-14T09:03:00.127 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_present cluster1 mirror image_32 deleted '' 2024-09-14T09:03:00.127 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:03:00.128 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:03:00.128 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_32 2024-09-14T09:03:00.128 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=deleted 2024-09-14T09:03:00.128 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:03:00.128 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=deleted 2024-09-14T09:03:00.128 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_image_id 2024-09-14T09:03:00.128 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_32 2024-09-14T09:03:00.128 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:03:00.128 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:03:00.128 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_32 2024-09-14T09:03:00.128 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_32 2024-09-14T09:03:00.128 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:03:00.162 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_32: (2) No such file or directory 2024-09-14T09:03:00.166 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_image_id= 2024-09-14T09:03:00.166 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:03:00.166 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test deleted = deleted 2024-09-14T09:03:00.167 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:03:00.167 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:03:00.167 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_33 2024-09-14T09:03:00.167 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_present cluster1 mirror image_33 deleted 2024-09-14T09:03:00.167 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:03:00.167 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:03:00.167 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_33 2024-09-14T09:03:00.167 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=deleted 2024-09-14T09:03:00.167 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:03:00.167 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T09:03:00.167 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:03:00.167 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_33 2024-09-14T09:03:00.167 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:03:00.168 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:03:00.168 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_33 2024-09-14T09:03:00.168 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_33 2024-09-14T09:03:00.168 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:03:00.201 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_33: (2) No such file or directory 2024-09-14T09:03:00.204 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image_id= 2024-09-14T09:03:00.204 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T09:03:00.204 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.1 2024-09-14T09:03:00.306 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_present cluster1 mirror image_33 deleted '' 2024-09-14T09:03:00.306 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:03:00.306 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:03:00.306 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_33 2024-09-14T09:03:00.306 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=deleted 2024-09-14T09:03:00.306 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:03:00.306 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=deleted 2024-09-14T09:03:00.306 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_image_id 2024-09-14T09:03:00.306 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_33 2024-09-14T09:03:00.306 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:03:00.307 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:03:00.307 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_33 2024-09-14T09:03:00.307 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_33 2024-09-14T09:03:00.307 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:03:00.341 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_33: (2) No such file or directory 2024-09-14T09:03:00.345 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_image_id= 2024-09-14T09:03:00.345 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:03:00.345 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test deleted = deleted 2024-09-14T09:03:00.345 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:03:00.345 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:03:00.345 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_34 2024-09-14T09:03:00.345 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_present cluster1 mirror image_34 deleted 2024-09-14T09:03:00.345 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:03:00.346 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:03:00.346 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_34 2024-09-14T09:03:00.346 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=deleted 2024-09-14T09:03:00.346 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:03:00.346 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T09:03:00.346 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:03:00.346 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_34 2024-09-14T09:03:00.346 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:03:00.346 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:03:00.346 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_34 2024-09-14T09:03:00.346 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_34 2024-09-14T09:03:00.346 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:03:00.380 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_34: (2) No such file or directory 2024-09-14T09:03:00.383 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image_id= 2024-09-14T09:03:00.383 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T09:03:00.383 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.1 2024-09-14T09:03:00.485 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_present cluster1 mirror image_34 deleted '' 2024-09-14T09:03:00.486 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:03:00.486 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:03:00.486 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_34 2024-09-14T09:03:00.486 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=deleted 2024-09-14T09:03:00.486 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:03:00.486 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=deleted 2024-09-14T09:03:00.486 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_image_id 2024-09-14T09:03:00.486 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_34 2024-09-14T09:03:00.486 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:03:00.486 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:03:00.486 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_34 2024-09-14T09:03:00.487 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_34 2024-09-14T09:03:00.487 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:03:00.521 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_34: (2) No such file or directory 2024-09-14T09:03:00.526 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_image_id= 2024-09-14T09:03:00.526 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:03:00.526 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test deleted = deleted 2024-09-14T09:03:00.526 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:03:00.526 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:03:00.526 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_35 2024-09-14T09:03:00.526 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_present cluster1 mirror image_35 deleted 2024-09-14T09:03:00.526 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:03:00.526 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:03:00.527 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_35 2024-09-14T09:03:00.527 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=deleted 2024-09-14T09:03:00.527 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:03:00.527 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T09:03:00.527 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:03:00.527 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_35 2024-09-14T09:03:00.527 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:03:00.527 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:03:00.527 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_35 2024-09-14T09:03:00.527 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_35 2024-09-14T09:03:00.527 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:03:00.563 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_35: (2) No such file or directory 2024-09-14T09:03:00.566 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image_id= 2024-09-14T09:03:00.566 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T09:03:00.567 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.1 2024-09-14T09:03:00.668 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_present cluster1 mirror image_35 deleted '' 2024-09-14T09:03:00.669 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:03:00.669 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:03:00.669 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_35 2024-09-14T09:03:00.669 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=deleted 2024-09-14T09:03:00.669 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:03:00.669 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=deleted 2024-09-14T09:03:00.669 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_image_id 2024-09-14T09:03:00.669 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_35 2024-09-14T09:03:00.669 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:03:00.669 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:03:00.669 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_35 2024-09-14T09:03:00.670 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_35 2024-09-14T09:03:00.670 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:03:00.707 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_35: (2) No such file or directory 2024-09-14T09:03:00.712 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_image_id= 2024-09-14T09:03:00.712 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:03:00.712 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test deleted = deleted 2024-09-14T09:03:00.713 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:03:00.713 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:03:00.713 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_36 2024-09-14T09:03:00.713 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_present cluster1 mirror image_36 deleted 2024-09-14T09:03:00.713 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:03:00.713 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:03:00.713 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_36 2024-09-14T09:03:00.713 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=deleted 2024-09-14T09:03:00.713 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:03:00.713 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T09:03:00.713 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:03:00.713 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_36 2024-09-14T09:03:00.713 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:03:00.714 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:03:00.714 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_36 2024-09-14T09:03:00.714 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_36 2024-09-14T09:03:00.714 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:03:00.746 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_36: (2) No such file or directory 2024-09-14T09:03:00.749 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image_id= 2024-09-14T09:03:00.749 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T09:03:00.750 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.1 2024-09-14T09:03:00.851 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_present cluster1 mirror image_36 deleted '' 2024-09-14T09:03:00.851 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:03:00.851 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:03:00.851 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_36 2024-09-14T09:03:00.852 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=deleted 2024-09-14T09:03:00.852 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:03:00.852 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=deleted 2024-09-14T09:03:00.852 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_image_id 2024-09-14T09:03:00.852 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_36 2024-09-14T09:03:00.852 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:03:00.852 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:03:00.852 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_36 2024-09-14T09:03:00.852 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_36 2024-09-14T09:03:00.852 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:03:00.888 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_36: (2) No such file or directory 2024-09-14T09:03:00.892 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_image_id= 2024-09-14T09:03:00.892 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:03:00.892 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test deleted = deleted 2024-09-14T09:03:00.892 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:03:00.892 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:03:00.893 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_37 2024-09-14T09:03:00.893 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_present cluster1 mirror image_37 deleted 2024-09-14T09:03:00.893 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:03:00.893 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:03:00.893 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_37 2024-09-14T09:03:00.893 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=deleted 2024-09-14T09:03:00.893 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:03:00.893 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T09:03:00.893 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:03:00.893 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_37 2024-09-14T09:03:00.893 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:03:00.893 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:03:00.893 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_37 2024-09-14T09:03:00.893 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_37 2024-09-14T09:03:00.893 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:03:00.926 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_37: (2) No such file or directory 2024-09-14T09:03:00.930 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image_id= 2024-09-14T09:03:00.930 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T09:03:00.930 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.1 2024-09-14T09:03:01.031 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_present cluster1 mirror image_37 deleted '' 2024-09-14T09:03:01.032 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:03:01.032 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:03:01.032 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_37 2024-09-14T09:03:01.032 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=deleted 2024-09-14T09:03:01.032 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:03:01.032 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=deleted 2024-09-14T09:03:01.032 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_image_id 2024-09-14T09:03:01.032 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_37 2024-09-14T09:03:01.032 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:03:01.033 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:03:01.033 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_37 2024-09-14T09:03:01.033 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_37 2024-09-14T09:03:01.033 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:03:01.068 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_37: (2) No such file or directory 2024-09-14T09:03:01.071 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_image_id= 2024-09-14T09:03:01.072 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:03:01.072 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test deleted = deleted 2024-09-14T09:03:01.072 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:03:01.072 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:03:01.072 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_38 2024-09-14T09:03:01.072 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_present cluster1 mirror image_38 deleted 2024-09-14T09:03:01.072 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:03:01.072 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:03:01.072 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_38 2024-09-14T09:03:01.072 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=deleted 2024-09-14T09:03:01.073 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:03:01.073 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T09:03:01.073 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:03:01.073 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_38 2024-09-14T09:03:01.073 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:03:01.073 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:03:01.073 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_38 2024-09-14T09:03:01.073 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_38 2024-09-14T09:03:01.073 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:03:01.105 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_38: (2) No such file or directory 2024-09-14T09:03:01.108 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image_id= 2024-09-14T09:03:01.108 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T09:03:01.109 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.1 2024-09-14T09:03:01.210 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_present cluster1 mirror image_38 deleted '' 2024-09-14T09:03:01.210 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:03:01.210 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:03:01.210 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_38 2024-09-14T09:03:01.210 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=deleted 2024-09-14T09:03:01.211 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:03:01.211 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=deleted 2024-09-14T09:03:01.211 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_image_id 2024-09-14T09:03:01.211 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_38 2024-09-14T09:03:01.211 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:03:01.211 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:03:01.211 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_38 2024-09-14T09:03:01.211 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_38 2024-09-14T09:03:01.211 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:03:01.446 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_38: (2) No such file or directory 2024-09-14T09:03:01.450 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_image_id= 2024-09-14T09:03:01.450 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:03:01.450 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test deleted = deleted 2024-09-14T09:03:01.450 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:03:01.451 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:03:01.451 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_39 2024-09-14T09:03:01.451 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_present cluster1 mirror image_39 deleted 2024-09-14T09:03:01.451 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:03:01.451 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:03:01.451 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_39 2024-09-14T09:03:01.451 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=deleted 2024-09-14T09:03:01.451 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:03:01.451 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T09:03:01.451 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:03:01.451 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_39 2024-09-14T09:03:01.451 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:03:01.451 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:03:01.451 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_39 2024-09-14T09:03:01.452 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_39 2024-09-14T09:03:01.452 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:03:01.488 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_39: (2) No such file or directory 2024-09-14T09:03:01.491 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image_id= 2024-09-14T09:03:01.491 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T09:03:01.491 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.1 2024-09-14T09:03:01.593 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_present cluster1 mirror image_39 deleted '' 2024-09-14T09:03:01.593 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:03:01.593 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:03:01.593 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_39 2024-09-14T09:03:01.593 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=deleted 2024-09-14T09:03:01.593 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:03:01.594 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=deleted 2024-09-14T09:03:01.594 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_image_id 2024-09-14T09:03:01.594 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_39 2024-09-14T09:03:01.594 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:03:01.594 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:03:01.594 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_39 2024-09-14T09:03:01.594 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_39 2024-09-14T09:03:01.594 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:03:01.629 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_39: (2) No such file or directory 2024-09-14T09:03:01.633 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_image_id= 2024-09-14T09:03:01.633 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:03:01.633 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test deleted = deleted 2024-09-14T09:03:01.634 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:03:01.634 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:03:01.634 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_40 2024-09-14T09:03:01.634 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_present cluster1 mirror image_40 deleted 2024-09-14T09:03:01.634 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:03:01.634 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:03:01.634 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_40 2024-09-14T09:03:01.634 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=deleted 2024-09-14T09:03:01.634 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:03:01.634 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T09:03:01.634 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:03:01.634 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_40 2024-09-14T09:03:01.634 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:03:01.635 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:03:01.635 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_40 2024-09-14T09:03:01.635 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_40 2024-09-14T09:03:01.635 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:03:01.668 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_40: (2) No such file or directory 2024-09-14T09:03:01.671 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image_id= 2024-09-14T09:03:01.671 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T09:03:01.671 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.1 2024-09-14T09:03:01.773 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_present cluster1 mirror image_40 deleted '' 2024-09-14T09:03:01.773 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:03:01.773 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:03:01.773 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_40 2024-09-14T09:03:01.773 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=deleted 2024-09-14T09:03:01.773 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:03:01.773 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=deleted 2024-09-14T09:03:01.773 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_image_id 2024-09-14T09:03:01.774 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_40 2024-09-14T09:03:01.774 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:03:01.774 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:03:01.774 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_40 2024-09-14T09:03:01.774 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_40 2024-09-14T09:03:01.774 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:03:01.807 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_40: (2) No such file or directory 2024-09-14T09:03:01.812 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_image_id= 2024-09-14T09:03:01.812 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:03:01.812 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test deleted = deleted 2024-09-14T09:03:01.812 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:03:01.812 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:03:01.812 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_41 2024-09-14T09:03:01.812 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_present cluster1 mirror image_41 deleted 2024-09-14T09:03:01.812 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:03:01.813 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:03:01.813 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_41 2024-09-14T09:03:01.813 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=deleted 2024-09-14T09:03:01.813 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:03:01.813 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T09:03:01.813 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:03:01.813 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_41 2024-09-14T09:03:01.813 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:03:01.813 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:03:01.813 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_41 2024-09-14T09:03:01.813 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_41 2024-09-14T09:03:01.813 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:03:01.846 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_41: (2) No such file or directory 2024-09-14T09:03:01.849 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image_id= 2024-09-14T09:03:01.849 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T09:03:01.849 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.1 2024-09-14T09:03:01.951 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_present cluster1 mirror image_41 deleted '' 2024-09-14T09:03:01.951 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:03:01.951 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:03:01.951 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_41 2024-09-14T09:03:01.951 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=deleted 2024-09-14T09:03:01.951 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:03:01.951 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=deleted 2024-09-14T09:03:01.951 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_image_id 2024-09-14T09:03:01.952 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_41 2024-09-14T09:03:01.952 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:03:01.952 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:03:01.952 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_41 2024-09-14T09:03:01.952 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_41 2024-09-14T09:03:01.952 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:03:01.987 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_41: (2) No such file or directory 2024-09-14T09:03:01.992 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_image_id= 2024-09-14T09:03:01.992 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:03:01.992 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test deleted = deleted 2024-09-14T09:03:01.992 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:03:01.992 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:03:01.992 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_42 2024-09-14T09:03:01.992 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_present cluster1 mirror image_42 deleted 2024-09-14T09:03:01.992 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:03:01.992 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:03:01.993 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_42 2024-09-14T09:03:01.993 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=deleted 2024-09-14T09:03:01.993 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:03:01.993 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T09:03:01.993 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:03:01.993 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_42 2024-09-14T09:03:01.993 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:03:01.993 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:03:01.993 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_42 2024-09-14T09:03:01.993 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_42 2024-09-14T09:03:01.994 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:03:02.026 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_42: (2) No such file or directory 2024-09-14T09:03:02.029 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image_id= 2024-09-14T09:03:02.029 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T09:03:02.029 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.1 2024-09-14T09:03:02.130 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_present cluster1 mirror image_42 deleted '' 2024-09-14T09:03:02.130 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:03:02.130 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:03:02.130 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_42 2024-09-14T09:03:02.131 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=deleted 2024-09-14T09:03:02.131 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:03:02.131 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=deleted 2024-09-14T09:03:02.131 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_image_id 2024-09-14T09:03:02.131 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_42 2024-09-14T09:03:02.131 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:03:02.131 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:03:02.131 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_42 2024-09-14T09:03:02.131 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_42 2024-09-14T09:03:02.131 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:03:02.165 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_42: (2) No such file or directory 2024-09-14T09:03:02.169 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_image_id= 2024-09-14T09:03:02.169 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:03:02.169 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test deleted = deleted 2024-09-14T09:03:02.170 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:03:02.170 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:03:02.170 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_43 2024-09-14T09:03:02.170 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_present cluster1 mirror image_43 deleted 2024-09-14T09:03:02.170 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:03:02.170 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:03:02.170 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_43 2024-09-14T09:03:02.170 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=deleted 2024-09-14T09:03:02.170 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:03:02.170 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T09:03:02.170 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:03:02.170 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_43 2024-09-14T09:03:02.171 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:03:02.171 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:03:02.171 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_43 2024-09-14T09:03:02.171 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_43 2024-09-14T09:03:02.171 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:03:02.203 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_43: (2) No such file or directory 2024-09-14T09:03:02.207 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image_id= 2024-09-14T09:03:02.207 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T09:03:02.207 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.1 2024-09-14T09:03:02.308 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_present cluster1 mirror image_43 deleted '' 2024-09-14T09:03:02.309 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:03:02.309 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:03:02.309 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_43 2024-09-14T09:03:02.309 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=deleted 2024-09-14T09:03:02.309 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:03:02.309 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=deleted 2024-09-14T09:03:02.309 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_image_id 2024-09-14T09:03:02.309 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_43 2024-09-14T09:03:02.309 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:03:02.309 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:03:02.309 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_43 2024-09-14T09:03:02.309 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_43 2024-09-14T09:03:02.309 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:03:02.343 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_43: (2) No such file or directory 2024-09-14T09:03:02.348 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_image_id= 2024-09-14T09:03:02.348 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:03:02.348 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test deleted = deleted 2024-09-14T09:03:02.348 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:03:02.348 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:03:02.348 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_44 2024-09-14T09:03:02.348 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_present cluster1 mirror image_44 deleted 2024-09-14T09:03:02.348 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:03:02.348 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:03:02.348 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_44 2024-09-14T09:03:02.348 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=deleted 2024-09-14T09:03:02.349 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:03:02.349 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T09:03:02.349 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:03:02.349 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_44 2024-09-14T09:03:02.349 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:03:02.349 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:03:02.349 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_44 2024-09-14T09:03:02.349 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_44 2024-09-14T09:03:02.349 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:03:02.383 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_44: (2) No such file or directory 2024-09-14T09:03:02.386 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image_id= 2024-09-14T09:03:02.386 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T09:03:02.386 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.1 2024-09-14T09:03:02.488 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_present cluster1 mirror image_44 deleted '' 2024-09-14T09:03:02.488 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:03:02.489 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:03:02.489 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_44 2024-09-14T09:03:02.489 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=deleted 2024-09-14T09:03:02.489 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:03:02.489 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=deleted 2024-09-14T09:03:02.489 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_image_id 2024-09-14T09:03:02.489 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_44 2024-09-14T09:03:02.489 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:03:02.489 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:03:02.489 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_44 2024-09-14T09:03:02.490 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_44 2024-09-14T09:03:02.490 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:03:02.523 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_44: (2) No such file or directory 2024-09-14T09:03:02.528 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_image_id= 2024-09-14T09:03:02.528 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:03:02.528 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test deleted = deleted 2024-09-14T09:03:02.528 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:03:02.528 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:03:02.528 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_45 2024-09-14T09:03:02.528 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_present cluster1 mirror image_45 deleted 2024-09-14T09:03:02.528 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:03:02.528 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:03:02.529 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_45 2024-09-14T09:03:02.529 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=deleted 2024-09-14T09:03:02.529 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:03:02.529 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T09:03:02.529 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:03:02.529 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_45 2024-09-14T09:03:02.529 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:03:02.529 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:03:02.529 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_45 2024-09-14T09:03:02.529 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_45 2024-09-14T09:03:02.529 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:03:02.562 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_45: (2) No such file or directory 2024-09-14T09:03:02.565 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image_id= 2024-09-14T09:03:02.565 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T09:03:02.566 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.1 2024-09-14T09:03:02.668 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_present cluster1 mirror image_45 deleted '' 2024-09-14T09:03:02.668 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:03:02.668 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:03:02.668 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_45 2024-09-14T09:03:02.668 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=deleted 2024-09-14T09:03:02.668 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:03:02.668 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=deleted 2024-09-14T09:03:02.668 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_image_id 2024-09-14T09:03:02.668 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_45 2024-09-14T09:03:02.668 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:03:02.668 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:03:02.668 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_45 2024-09-14T09:03:02.669 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_45 2024-09-14T09:03:02.669 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:03:02.704 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_45: (2) No such file or directory 2024-09-14T09:03:02.709 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_image_id= 2024-09-14T09:03:02.709 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:03:02.709 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test deleted = deleted 2024-09-14T09:03:02.709 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:03:02.709 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:03:02.709 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_46 2024-09-14T09:03:02.709 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_present cluster1 mirror image_46 deleted 2024-09-14T09:03:02.709 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:03:02.710 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:03:02.710 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_46 2024-09-14T09:03:02.710 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=deleted 2024-09-14T09:03:02.710 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:03:02.710 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T09:03:02.710 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:03:02.710 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_46 2024-09-14T09:03:02.710 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:03:02.710 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:03:02.710 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_46 2024-09-14T09:03:02.711 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_46 2024-09-14T09:03:02.711 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:03:02.743 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_46: (2) No such file or directory 2024-09-14T09:03:02.746 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image_id= 2024-09-14T09:03:02.746 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T09:03:02.746 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.1 2024-09-14T09:03:02.848 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_present cluster1 mirror image_46 deleted '' 2024-09-14T09:03:02.848 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:03:02.848 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:03:02.848 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_46 2024-09-14T09:03:02.848 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=deleted 2024-09-14T09:03:02.849 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:03:02.849 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=deleted 2024-09-14T09:03:02.849 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_image_id 2024-09-14T09:03:02.849 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_46 2024-09-14T09:03:02.849 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:03:02.849 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:03:02.849 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_46 2024-09-14T09:03:02.849 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_46 2024-09-14T09:03:02.849 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:03:02.883 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_46: (2) No such file or directory 2024-09-14T09:03:02.888 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_image_id= 2024-09-14T09:03:02.888 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:03:02.888 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test deleted = deleted 2024-09-14T09:03:02.888 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:03:02.888 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:03:02.888 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_47 2024-09-14T09:03:02.888 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_present cluster1 mirror image_47 deleted 2024-09-14T09:03:02.888 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:03:02.888 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:03:02.888 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_47 2024-09-14T09:03:02.889 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=deleted 2024-09-14T09:03:02.889 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:03:02.889 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T09:03:02.889 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:03:02.889 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_47 2024-09-14T09:03:02.889 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:03:02.889 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:03:02.889 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_47 2024-09-14T09:03:02.889 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_47 2024-09-14T09:03:02.889 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:03:02.922 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_47: (2) No such file or directory 2024-09-14T09:03:02.925 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image_id= 2024-09-14T09:03:02.925 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T09:03:02.925 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.1 2024-09-14T09:03:03.027 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_present cluster1 mirror image_47 deleted '' 2024-09-14T09:03:03.027 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:03:03.028 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:03:03.028 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_47 2024-09-14T09:03:03.028 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=deleted 2024-09-14T09:03:03.028 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:03:03.028 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=deleted 2024-09-14T09:03:03.028 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_image_id 2024-09-14T09:03:03.028 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_47 2024-09-14T09:03:03.028 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:03:03.028 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:03:03.028 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_47 2024-09-14T09:03:03.029 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_47 2024-09-14T09:03:03.029 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:03:03.061 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_47: (2) No such file or directory 2024-09-14T09:03:03.066 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_image_id= 2024-09-14T09:03:03.066 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:03:03.066 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test deleted = deleted 2024-09-14T09:03:03.066 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:03:03.066 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:03:03.066 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_48 2024-09-14T09:03:03.066 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_present cluster1 mirror image_48 deleted 2024-09-14T09:03:03.066 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:03:03.066 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:03:03.066 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_48 2024-09-14T09:03:03.067 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=deleted 2024-09-14T09:03:03.067 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:03:03.067 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T09:03:03.067 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:03:03.067 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_48 2024-09-14T09:03:03.067 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:03:03.067 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:03:03.067 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_48 2024-09-14T09:03:03.067 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_48 2024-09-14T09:03:03.067 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:03:03.100 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_48: (2) No such file or directory 2024-09-14T09:03:03.103 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image_id= 2024-09-14T09:03:03.103 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T09:03:03.103 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.1 2024-09-14T09:03:03.205 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_present cluster1 mirror image_48 deleted '' 2024-09-14T09:03:03.205 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:03:03.205 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:03:03.205 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_48 2024-09-14T09:03:03.205 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=deleted 2024-09-14T09:03:03.205 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:03:03.206 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=deleted 2024-09-14T09:03:03.206 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_image_id 2024-09-14T09:03:03.206 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_48 2024-09-14T09:03:03.206 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:03:03.206 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:03:03.206 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_48 2024-09-14T09:03:03.206 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_48 2024-09-14T09:03:03.206 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:03:03.239 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_48: (2) No such file or directory 2024-09-14T09:03:03.244 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_image_id= 2024-09-14T09:03:03.244 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:03:03.244 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test deleted = deleted 2024-09-14T09:03:03.244 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:03:03.244 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:03:03.244 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_49 2024-09-14T09:03:03.244 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_present cluster1 mirror image_49 deleted 2024-09-14T09:03:03.244 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:03:03.244 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:03:03.244 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_49 2024-09-14T09:03:03.244 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=deleted 2024-09-14T09:03:03.245 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:03:03.245 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T09:03:03.245 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:03:03.245 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_49 2024-09-14T09:03:03.245 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:03:03.245 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:03:03.245 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_49 2024-09-14T09:03:03.245 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_49 2024-09-14T09:03:03.245 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:03:03.278 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_49: (2) No such file or directory 2024-09-14T09:03:03.281 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image_id= 2024-09-14T09:03:03.281 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T09:03:03.281 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.1 2024-09-14T09:03:03.383 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_present cluster1 mirror image_49 deleted '' 2024-09-14T09:03:03.383 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:03:03.383 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:03:03.383 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_49 2024-09-14T09:03:03.383 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=deleted 2024-09-14T09:03:03.383 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:03:03.383 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=deleted 2024-09-14T09:03:03.383 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_image_id 2024-09-14T09:03:03.383 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_49 2024-09-14T09:03:03.383 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:03:03.384 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:03:03.384 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_49 2024-09-14T09:03:03.384 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_49 2024-09-14T09:03:03.384 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:03:03.417 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_49: (2) No such file or directory 2024-09-14T09:03:03.422 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_image_id= 2024-09-14T09:03:03.422 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:03:03.422 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test deleted = deleted 2024-09-14T09:03:03.422 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:03:03.422 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T09:03:03.422 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_50 2024-09-14T09:03:03.422 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_present cluster1 mirror image_50 deleted 2024-09-14T09:03:03.422 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:03:03.422 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:03:03.422 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_50 2024-09-14T09:03:03.423 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=deleted 2024-09-14T09:03:03.423 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:03:03.423 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T09:03:03.423 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:03:03.423 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_50 2024-09-14T09:03:03.423 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:03:03.423 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:03:03.423 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_50 2024-09-14T09:03:03.423 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_50 2024-09-14T09:03:03.423 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:03:03.456 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_50: (2) No such file or directory 2024-09-14T09:03:03.459 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image_id= 2024-09-14T09:03:03.459 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T09:03:03.459 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.1 2024-09-14T09:03:03.561 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_present cluster1 mirror image_50 deleted '' 2024-09-14T09:03:03.561 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:03:03.561 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:03:03.561 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_50 2024-09-14T09:03:03.561 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=deleted 2024-09-14T09:03:03.562 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:03:03.562 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=deleted 2024-09-14T09:03:03.562 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_image_id 2024-09-14T09:03:03.562 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_50 2024-09-14T09:03:03.562 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:03:03.562 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:03:03.562 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_50 2024-09-14T09:03:03.562 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_50 2024-09-14T09:03:03.562 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:03:03.596 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_50: (2) No such file or directory 2024-09-14T09:03:03.599 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_image_id= 2024-09-14T09:03:03.600 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:03:03.600 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test deleted = deleted 2024-09-14T09:03:03.600 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:03:03.600 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ testlog 'TEST: delete images during bootstrap' 2024-09-14T09:03:03.600 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ tee -a /tmp/tmp.IUd9LESfOA/rbd-mirror.test.log 2024-09-14T09:03:03.600 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ date '+%F %T' 2024-09-14T09:03:03.601 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ echo 2024-09-14 09:03:03 TEST: delete images during bootstrap 2024-09-14T09:03:03.601 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:2024-09-14 09:03:03 TEST: delete images during bootstrap 2024-09-14T09:03:03.601 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set_pool_mirror_mode cluster1 mirror image 2024-09-14T09:03:03.601 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:03:03.601 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:03:03.601 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=image 2024-09-14T09:03:03.601 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster1 mirror pool enable mirror image 2024-09-14T09:03:03.636 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd: mirroring is already configured for image mode 2024-09-14T09:03:03.639 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set_pool_mirror_mode cluster2 mirror image 2024-09-14T09:03:03.639 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:03:03.639 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:03:03.639 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=image 2024-09-14T09:03:03.639 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 mirror pool enable mirror image 2024-09-14T09:03:03.673 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd: mirroring is already configured for image mode 2024-09-14T09:03:03.680 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ start_mirror cluster1 2024-09-14T09:03:03.680 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:03:03.680 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local instance 2024-09-14T09:03:03.680 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set_cluster_instance cluster1 cluster instance 2024-09-14T09:03:03.680 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local val=cluster1 2024-09-14T09:03:03.680 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster_var_name=cluster 2024-09-14T09:03:03.680 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local instance_var_name=instance 2024-09-14T09:03:03.681 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cluster=cluster1 2024-09-14T09:03:03.681 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ instance=cluster1 2024-09-14T09:03:03.681 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' cluster1 = cluster1 ']' 2024-09-14T09:03:03.681 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ instance=0 2024-09-14T09:03:03.681 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ eval cluster=cluster1 2024-09-14T09:03:03.681 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ cluster=cluster1 2024-09-14T09:03:03.681 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ eval instance=0 2024-09-14T09:03:03.681 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ instance=0 2024-09-14T09:03:03.681 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n 1 2024-09-14T09:03:03.681 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 2024-09-14T09:03:03.681 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=test 2024-09-14T09:03:03.681 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ seq 1 10 2024-09-14T09:03:03.681 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 10` 2024-09-14T09:03:03.682 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_1 2024-09-14T09:03:03.682 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image cluster2 mirror image_1 512M 2024-09-14T09:03:03.682 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:03:03.682 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T09:03:03.682 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:03:03.682 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T09:03:03.682 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_1 2024-09-14T09:03:03.682 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T09:03:03.682 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local size=128 2024-09-14T09:03:03.682 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n 512M ']' 2024-09-14T09:03:03.682 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ size=512M 2024-09-14T09:03:03.682 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T09:03:03.682 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering mirror/image_1 2024-09-14T09:03:03.727 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ enable_mirror cluster2 mirror image_1 2024-09-14T09:03:03.727 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:03:03.727 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:03:03.727 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_1 2024-09-14T09:03:03.728 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T09:03:03.728 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_1 snapshot 2024-09-14T09:03:04.635 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Mirroring enabled 2024-09-14T09:03:04.642 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 info mirror/image_1 2024-09-14T09:03:04.683 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_1': 2024-09-14T09:03:04.683 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 512 MiB in 128 objects 2024-09-14T09:03:04.683 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T09:03:04.683 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 1 2024-09-14T09:03:04.684 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 18e9e92bc488 2024-09-14T09:03:04.684 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.18e9e92bc488 2024-09-14T09:03:04.684 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T09:03:04.684 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering 2024-09-14T09:03:04.684 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T09:03:04.684 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T09:03:04.684 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 09:03:03 2024 2024-09-14T09:03:04.684 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 09:03:03 2024 2024-09-14T09:03:04.684 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 09:03:03 2024 2024-09-14T09:03:04.684 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T09:03:04.684 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T09:03:04.684 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: 42f13ea6-c218-402f-aec8-46667a311cdc 2024-09-14T09:03:04.684 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: true 2024-09-14T09:03:04.687 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ stress_write_image cluster2 mirror image_1 2024-09-14T09:03:04.687 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:03:04.687 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:03:04.687 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_1 2024-09-14T09:03:04.687 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T09:03:04.689 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local duration=8 2024-09-14T09:03:04.689 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set +e 2024-09-14T09:03:04.689 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ timeout 8s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_1 --debug-rbd=20 --debug-journaler=20 2024-09-14T09:03:05.159 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:100 IOs, 75964416 bytes 2024-09-14T09:03:05.413 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:200 IOs, 136784896 bytes 2024-09-14T09:03:05.513 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:300 IOs, 142928896 bytes 2024-09-14T09:03:05.728 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:400 IOs, 202586112 bytes 2024-09-14T09:03:05.940 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:500 IOs, 261017600 bytes 2024-09-14T09:03:06.180 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:600 IOs, 341256192 bytes 2024-09-14T09:03:06.466 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:700 IOs, 419503104 bytes 2024-09-14T09:03:06.687 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:800 IOs, 481723392 bytes 2024-09-14T09:03:06.813 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:900 IOs, 499167232 bytes 2024-09-14T09:03:06.940 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1000 IOs, 519481344 bytes 2024-09-14T09:03:07.090 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1100 IOs, 551581696 bytes 2024-09-14T09:03:07.207 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1200 IOs, 569951232 bytes 2024-09-14T09:03:07.363 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1300 IOs, 612800512 bytes 2024-09-14T09:03:07.485 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1400 IOs, 620322816 bytes 2024-09-14T09:03:07.633 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1500 IOs, 646529024 bytes 2024-09-14T09:03:07.879 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1600 IOs, 733668352 bytes 2024-09-14T09:03:08.075 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1700 IOs, 790155264 bytes 2024-09-14T09:03:08.365 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1800 IOs, 862442496 bytes 2024-09-14T09:03:08.567 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1900 IOs, 927133696 bytes 2024-09-14T09:03:08.777 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2000 IOs, 968926208 bytes 2024-09-14T09:03:09.056 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2100 IOs, 1051606016 bytes 2024-09-14T09:03:09.208 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2200 IOs, 1082208256 bytes 2024-09-14T09:03:09.314 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2300 IOs, 1094814720 bytes 2024-09-14T09:03:09.513 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2400 IOs, 1145853952 bytes 2024-09-14T09:03:09.602 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2500 IOs, 1148299264 bytes 2024-09-14T09:03:09.692 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2600 IOs, 1167224832 bytes 2024-09-14T09:03:09.994 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2700 IOs, 1252688896 bytes 2024-09-14T09:03:10.184 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2800 IOs, 1301069824 bytes 2024-09-14T09:03:10.376 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2900 IOs, 1341346816 bytes 2024-09-14T09:03:10.472 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3000 IOs, 1353721856 bytes 2024-09-14T09:03:10.620 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3100 IOs, 1388252160 bytes 2024-09-14T09:03:10.863 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3200 IOs, 1445594112 bytes 2024-09-14T09:03:10.976 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3300 IOs, 1460055040 bytes 2024-09-14T09:03:11.053 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3400 IOs, 1462232064 bytes 2024-09-14T09:03:11.180 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3500 IOs, 1481180160 bytes 2024-09-14T09:03:11.307 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3600 IOs, 1501501440 bytes 2024-09-14T09:03:11.424 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3700 IOs, 1517411328 bytes 2024-09-14T09:03:11.527 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3800 IOs, 1533794304 bytes 2024-09-14T09:03:11.775 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3900 IOs, 1605390336 bytes 2024-09-14T09:03:11.878 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4000 IOs, 1608636416 bytes 2024-09-14T09:03:12.127 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4100 IOs, 1661865984 bytes 2024-09-14T09:03:12.186 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.0 2024-09-14T09:03:12.187 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Sent signal 15 2024-09-14T09:03:12.187 INFO:tasks.rbd_mirror_thrash:waiting for 8 secs before reviving daemons 2024-09-14T09:03:12.188 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi164.stderr:2024-09-14T09:03:12.187+0000 7f60ecd64640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 (PID: 53493) UID: 1000 2024-09-14T09:03:12.297 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4200 IOs, 1696667648 bytes 2024-09-14T09:03:12.584 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4300 IOs, 1777051648 bytes 2024-09-14T09:03:12.700 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ error_code=124 2024-09-14T09:03:12.701 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set -e 2024-09-14T09:03:12.701 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T09:03:12.701 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:03:12.701 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_present cluster1 mirror image_1 present 2024-09-14T09:03:12.701 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:03:12.701 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:03:12.701 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_1 2024-09-14T09:03:12.701 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=present 2024-09-14T09:03:12.701 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:03:12.701 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T09:03:12.701 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:03:12.701 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_1 2024-09-14T09:03:12.702 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:03:12.702 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:03:12.702 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_1 2024-09-14T09:03:12.702 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_1 2024-09-14T09:03:12.702 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:03:12.769 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image_id=13b2ae2a5c4f 2024-09-14T09:03:12.770 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T09:03:12.770 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.1 2024-09-14T09:03:12.871 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_present cluster1 mirror image_1 present 13b2ae2a5c4f 2024-09-14T09:03:12.871 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:03:12.871 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:03:12.872 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_1 2024-09-14T09:03:12.872 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=present 2024-09-14T09:03:12.872 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id=13b2ae2a5c4f 2024-09-14T09:03:12.872 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=deleted 2024-09-14T09:03:12.872 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_image_id 2024-09-14T09:03:12.872 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_1 2024-09-14T09:03:12.872 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:03:12.872 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:03:12.872 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_1 2024-09-14T09:03:12.872 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_1 2024-09-14T09:03:12.872 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:03:12.918 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_image_id=13b2ae2a5c4f 2024-09-14T09:03:12.918 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n 13b2ae2a5c4f 2024-09-14T09:03:12.918 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -z 13b2ae2a5c4f -o 13b2ae2a5c4f = 13b2ae2a5c4f 2024-09-14T09:03:12.918 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_state=present 2024-09-14T09:03:12.918 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test present = present 2024-09-14T09:03:12.918 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:03:12.918 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ disable_mirror cluster2 mirror image_1 2024-09-14T09:03:12.918 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:03:12.918 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:03:12.918 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_1 2024-09-14T09:03:12.919 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_1 2024-09-14T09:03:18.641 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi164.stderr:2024-09-14T09:03:18.640+0000 7f2517b34640 -1 rbd::mirror::ImageMap: 0x555e5006e240 handle_peer_ack_remove: failed to remove global_image_id=42f13ea6-c218-402f-aec8-46667a311cdc 2024-09-14T09:03:18.642 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Mirroring disabled 2024-09-14T09:03:18.650 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_present cluster1 mirror image_1 deleted 2024-09-14T09:03:18.651 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:03:18.651 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:03:18.651 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_1 2024-09-14T09:03:18.651 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=deleted 2024-09-14T09:03:18.651 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:03:18.651 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T09:03:18.651 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:03:18.651 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_1 2024-09-14T09:03:18.651 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:03:18.651 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:03:18.651 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_1 2024-09-14T09:03:18.651 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_1 2024-09-14T09:03:18.651 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:03:18.702 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image_id=13b2ae2a5c4f 2024-09-14T09:03:18.702 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T09:03:18.702 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.1 2024-09-14T09:03:18.804 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_present cluster1 mirror image_1 deleted 13b2ae2a5c4f 2024-09-14T09:03:18.804 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:03:18.804 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:03:18.804 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_1 2024-09-14T09:03:18.804 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=deleted 2024-09-14T09:03:18.805 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id=13b2ae2a5c4f 2024-09-14T09:03:18.805 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=deleted 2024-09-14T09:03:18.805 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_image_id 2024-09-14T09:03:18.805 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_1 2024-09-14T09:03:18.805 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:03:18.805 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:03:18.805 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_1 2024-09-14T09:03:18.805 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_1 2024-09-14T09:03:18.805 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:03:18.855 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_image_id=13b2ae2a5c4f 2024-09-14T09:03:18.855 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n 13b2ae2a5c4f 2024-09-14T09:03:18.855 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -z 13b2ae2a5c4f -o 13b2ae2a5c4f = 13b2ae2a5c4f 2024-09-14T09:03:18.855 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_state=present 2024-09-14T09:03:18.855 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test deleted = present 2024-09-14T09:03:18.856 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T09:03:18.856 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T09:03:19.854 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_present cluster1 mirror image_1 deleted 13b2ae2a5c4f 2024-09-14T09:03:19.854 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:03:19.855 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:03:19.855 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_1 2024-09-14T09:03:19.855 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=deleted 2024-09-14T09:03:19.855 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id=13b2ae2a5c4f 2024-09-14T09:03:19.855 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=deleted 2024-09-14T09:03:19.855 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_image_id 2024-09-14T09:03:19.855 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_1 2024-09-14T09:03:19.855 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:03:19.855 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:03:19.855 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_1 2024-09-14T09:03:19.855 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_1 2024-09-14T09:03:19.855 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:03:19.909 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_image_id=13b2ae2a5c4f 2024-09-14T09:03:19.909 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n 13b2ae2a5c4f 2024-09-14T09:03:19.909 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -z 13b2ae2a5c4f -o 13b2ae2a5c4f = 13b2ae2a5c4f 2024-09-14T09:03:19.910 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_state=present 2024-09-14T09:03:19.910 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test deleted = present 2024-09-14T09:03:19.910 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T09:03:19.910 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 2 2024-09-14T09:03:20.188 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.0 2024-09-14T09:03:20.188 INFO:teuthology.orchestra.run:waiting for 600 2024-09-14T09:03:20.189 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.0 2024-09-14T09:03:20.189 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Restarting daemon 2024-09-14T09:03:20.189 DEBUG:teuthology.orchestra.run.smithi164:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 2024-09-14T09:03:20.192 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Started 2024-09-14T09:03:20.192 INFO:tasks.rbd_mirror_thrash:waiting for 102 secs before thrashing 2024-09-14T09:03:21.911 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_present cluster1 mirror image_1 deleted 13b2ae2a5c4f 2024-09-14T09:03:21.911 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:03:21.911 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:03:21.911 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_1 2024-09-14T09:03:21.911 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=deleted 2024-09-14T09:03:21.911 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id=13b2ae2a5c4f 2024-09-14T09:03:21.911 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=deleted 2024-09-14T09:03:21.912 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_image_id 2024-09-14T09:03:21.912 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_1 2024-09-14T09:03:21.912 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:03:21.912 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:03:21.912 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_1 2024-09-14T09:03:21.912 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_1 2024-09-14T09:03:21.912 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:03:21.963 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_image_id=13b2ae2a5c4f 2024-09-14T09:03:21.964 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n 13b2ae2a5c4f 2024-09-14T09:03:21.964 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -z 13b2ae2a5c4f -o 13b2ae2a5c4f = 13b2ae2a5c4f 2024-09-14T09:03:21.964 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_state=present 2024-09-14T09:03:21.964 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test deleted = present 2024-09-14T09:03:21.964 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T09:03:21.964 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 4 2024-09-14T09:03:25.965 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_present cluster1 mirror image_1 deleted 13b2ae2a5c4f 2024-09-14T09:03:25.965 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:03:25.966 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:03:25.966 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_1 2024-09-14T09:03:25.966 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=deleted 2024-09-14T09:03:25.966 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id=13b2ae2a5c4f 2024-09-14T09:03:25.966 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=deleted 2024-09-14T09:03:25.966 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_image_id 2024-09-14T09:03:25.966 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_1 2024-09-14T09:03:25.966 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:03:25.966 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:03:25.966 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_1 2024-09-14T09:03:25.966 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_1 2024-09-14T09:03:25.966 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:03:26.021 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_image_id=13b2ae2a5c4f 2024-09-14T09:03:26.021 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n 13b2ae2a5c4f 2024-09-14T09:03:26.021 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -z 13b2ae2a5c4f -o 13b2ae2a5c4f = 13b2ae2a5c4f 2024-09-14T09:03:26.021 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_state=present 2024-09-14T09:03:26.021 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test deleted = present 2024-09-14T09:03:26.021 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T09:03:26.021 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 8 2024-09-14T09:03:34.023 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_present cluster1 mirror image_1 deleted 13b2ae2a5c4f 2024-09-14T09:03:34.023 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:03:34.023 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:03:34.023 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_1 2024-09-14T09:03:34.023 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=deleted 2024-09-14T09:03:34.023 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id=13b2ae2a5c4f 2024-09-14T09:03:34.023 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=deleted 2024-09-14T09:03:34.023 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_image_id 2024-09-14T09:03:34.023 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_1 2024-09-14T09:03:34.024 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:03:34.024 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:03:34.024 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_1 2024-09-14T09:03:34.024 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_1 2024-09-14T09:03:34.024 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:03:34.076 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_image_id=13b2ae2a5c4f 2024-09-14T09:03:34.076 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n 13b2ae2a5c4f 2024-09-14T09:03:34.076 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -z 13b2ae2a5c4f -o 13b2ae2a5c4f = 13b2ae2a5c4f 2024-09-14T09:03:34.076 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_state=present 2024-09-14T09:03:34.076 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test deleted = present 2024-09-14T09:03:34.076 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T09:03:34.076 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 8 2024-09-14T09:03:34.954 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.7355916260522865) > weight (0.25) 2024-09-14T09:03:34.954 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.1 2024-09-14T09:03:34.955 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Sent signal 15 2024-09-14T09:03:34.955 INFO:tasks.rbd_mirror_thrash:waiting for 0 secs before reviving daemons 2024-09-14T09:03:34.955 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.1 2024-09-14T09:03:34.955 INFO:teuthology.orchestra.run:waiting for 600 2024-09-14T09:03:34.956 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi164.stderr:2024-09-14T09:03:34.956+0000 7f7e39532640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 (PID: 52176) UID: 1000 2024-09-14T09:03:41.057 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.1 2024-09-14T09:03:41.057 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Restarting daemon 2024-09-14T09:03:41.058 DEBUG:teuthology.orchestra.run.smithi164:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 2024-09-14T09:03:41.061 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Started 2024-09-14T09:03:41.061 INFO:tasks.rbd_mirror_thrash:waiting for 119 secs before thrashing 2024-09-14T09:03:42.077 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_present cluster1 mirror image_1 deleted 13b2ae2a5c4f 2024-09-14T09:03:42.077 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:03:42.078 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:03:42.078 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_1 2024-09-14T09:03:42.078 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=deleted 2024-09-14T09:03:42.078 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id=13b2ae2a5c4f 2024-09-14T09:03:42.078 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=deleted 2024-09-14T09:03:42.078 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_image_id 2024-09-14T09:03:42.079 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_1 2024-09-14T09:03:42.079 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:03:42.079 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:03:42.079 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_1 2024-09-14T09:03:42.079 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_1 2024-09-14T09:03:42.079 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:03:42.125 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_image_id=13b2ae2a5c4f 2024-09-14T09:03:42.125 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n 13b2ae2a5c4f 2024-09-14T09:03:42.125 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -z 13b2ae2a5c4f -o 13b2ae2a5c4f = 13b2ae2a5c4f 2024-09-14T09:03:42.126 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_state=present 2024-09-14T09:03:42.126 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test deleted = present 2024-09-14T09:03:42.126 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T09:03:42.126 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 8 2024-09-14T09:03:50.127 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_present cluster1 mirror image_1 deleted 13b2ae2a5c4f 2024-09-14T09:03:50.128 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:03:50.128 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:03:50.128 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_1 2024-09-14T09:03:50.128 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=deleted 2024-09-14T09:03:50.128 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id=13b2ae2a5c4f 2024-09-14T09:03:50.128 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=deleted 2024-09-14T09:03:50.128 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_image_id 2024-09-14T09:03:50.128 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_1 2024-09-14T09:03:50.128 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:03:50.128 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:03:50.128 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_1 2024-09-14T09:03:50.128 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_1 2024-09-14T09:03:50.128 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:03:50.168 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_1: (2) No such file or directory 2024-09-14T09:03:50.172 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_image_id= 2024-09-14T09:03:50.172 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:03:50.173 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test deleted = deleted 2024-09-14T09:03:50.173 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:03:50.173 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ purge_snapshots cluster2 mirror image_1 2024-09-14T09:03:50.173 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:03:50.173 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:03:50.173 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_1 2024-09-14T09:03:50.173 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap purge mirror/image_1 2024-09-14T09:03:50.222 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image_retry cluster2 mirror image_1 2024-09-14T09:03:50.222 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:03:50.223 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:03:50.223 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_1 2024-09-14T09:03:50.223 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:03:50.223 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0 2024-09-14T09:03:50.224 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_1 2024-09-14T09:03:50.224 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:03:50.225 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:03:50.225 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_1 2024-09-14T09:03:50.225 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_1 2024-09-14T09:03:50.270 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_1 2024-09-14T09:03:50.482 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-14T09:03:50.486 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:03:50.486 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 10` 2024-09-14T09:03:50.487 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_2 2024-09-14T09:03:50.487 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image cluster2 mirror image_2 512M 2024-09-14T09:03:50.487 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:03:50.487 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T09:03:50.487 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:03:50.487 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T09:03:50.487 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_2 2024-09-14T09:03:50.487 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T09:03:50.487 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local size=128 2024-09-14T09:03:50.487 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n 512M ']' 2024-09-14T09:03:50.488 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ size=512M 2024-09-14T09:03:50.488 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T09:03:50.488 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering mirror/image_2 2024-09-14T09:03:50.536 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ enable_mirror cluster2 mirror image_2 2024-09-14T09:03:50.536 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:03:50.536 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:03:50.537 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_2 2024-09-14T09:03:50.537 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T09:03:50.537 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_2 snapshot 2024-09-14T09:03:50.684 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Mirroring enabled 2024-09-14T09:03:50.690 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 info mirror/image_2 2024-09-14T09:03:50.729 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_2': 2024-09-14T09:03:50.729 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 512 MiB in 128 objects 2024-09-14T09:03:50.729 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T09:03:50.729 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 1 2024-09-14T09:03:50.729 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 19105f12b948 2024-09-14T09:03:50.729 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.19105f12b948 2024-09-14T09:03:50.729 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T09:03:50.729 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering 2024-09-14T09:03:50.729 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T09:03:50.729 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T09:03:50.730 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 09:03:50 2024 2024-09-14T09:03:50.730 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 09:03:50 2024 2024-09-14T09:03:50.730 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 09:03:50 2024 2024-09-14T09:03:50.730 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T09:03:50.730 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T09:03:50.730 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: cda83485-3162-481b-ae8b-310b47a00e09 2024-09-14T09:03:50.730 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: true 2024-09-14T09:03:50.732 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ stress_write_image cluster2 mirror image_2 2024-09-14T09:03:50.732 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:03:50.732 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:03:50.733 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_2 2024-09-14T09:03:50.733 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T09:03:50.734 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local duration=12 2024-09-14T09:03:50.734 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set +e 2024-09-14T09:03:50.735 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ timeout 12s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_2 --debug-rbd=20 --debug-journaler=20 2024-09-14T09:03:50.971 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:100 IOs, 53694464 bytes 2024-09-14T09:03:51.164 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:200 IOs, 100867072 bytes 2024-09-14T09:03:51.291 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:300 IOs, 120559616 bytes 2024-09-14T09:03:51.449 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:400 IOs, 156862464 bytes 2024-09-14T09:03:51.681 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:500 IOs, 220596224 bytes 2024-09-14T09:03:51.955 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:600 IOs, 317096960 bytes 2024-09-14T09:03:52.067 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:700 IOs, 317712384 bytes 2024-09-14T09:03:52.198 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:800 IOs, 341289984 bytes 2024-09-14T09:03:52.329 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:900 IOs, 365243392 bytes 2024-09-14T09:03:52.603 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1000 IOs, 452367360 bytes 2024-09-14T09:03:52.779 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1100 IOs, 493751296 bytes 2024-09-14T09:03:52.910 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1200 IOs, 514167808 bytes 2024-09-14T09:03:53.054 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1300 IOs, 539677696 bytes 2024-09-14T09:03:53.157 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1400 IOs, 557720576 bytes 2024-09-14T09:03:53.410 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1500 IOs, 628624384 bytes 2024-09-14T09:03:53.559 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1600 IOs, 652887040 bytes 2024-09-14T09:03:53.717 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1700 IOs, 697722880 bytes 2024-09-14T09:03:53.886 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1800 IOs, 723918848 bytes 2024-09-14T09:03:53.983 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1900 IOs, 732318720 bytes 2024-09-14T09:03:54.111 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2000 IOs, 753944576 bytes 2024-09-14T09:03:54.355 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2100 IOs, 830486528 bytes 2024-09-14T09:03:54.502 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2200 IOs, 849811456 bytes 2024-09-14T09:03:54.615 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2300 IOs, 865550336 bytes 2024-09-14T09:03:54.808 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2400 IOs, 916040704 bytes 2024-09-14T09:03:54.948 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2500 IOs, 942707712 bytes 2024-09-14T09:03:55.138 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2600 IOs, 988956672 bytes 2024-09-14T09:03:55.261 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2700 IOs, 1003294720 bytes 2024-09-14T09:03:55.384 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2800 IOs, 1023281152 bytes 2024-09-14T09:03:55.620 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2900 IOs, 1088357376 bytes 2024-09-14T09:03:55.757 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3000 IOs, 1121405952 bytes 2024-09-14T09:03:55.925 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3100 IOs, 1148090368 bytes 2024-09-14T09:03:56.091 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3200 IOs, 1183695872 bytes 2024-09-14T09:03:56.200 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3300 IOs, 1217664000 bytes 2024-09-14T09:03:56.480 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3400 IOs, 1279343616 bytes 2024-09-14T09:03:56.576 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3500 IOs, 1286331392 bytes 2024-09-14T09:03:56.734 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3600 IOs, 1318971392 bytes 2024-09-14T09:03:56.922 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3700 IOs, 1386207232 bytes 2024-09-14T09:03:57.134 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3800 IOs, 1418408960 bytes 2024-09-14T09:03:57.292 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3900 IOs, 1451766784 bytes 2024-09-14T09:03:57.494 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4000 IOs, 1503312896 bytes 2024-09-14T09:03:57.761 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4100 IOs, 1581630464 bytes 2024-09-14T09:03:57.874 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4200 IOs, 1596910592 bytes 2024-09-14T09:03:58.021 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4300 IOs, 1628509184 bytes 2024-09-14T09:03:58.270 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4400 IOs, 1704162304 bytes 2024-09-14T09:03:58.465 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4500 IOs, 1747279872 bytes 2024-09-14T09:03:58.600 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4600 IOs, 1770764288 bytes 2024-09-14T09:03:58.697 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4700 IOs, 1778213888 bytes 2024-09-14T09:03:58.849 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4800 IOs, 1824172032 bytes 2024-09-14T09:03:59.023 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4900 IOs, 1851887616 bytes 2024-09-14T09:03:59.165 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5000 IOs, 1878226944 bytes 2024-09-14T09:03:59.290 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5100 IOs, 1898484736 bytes 2024-09-14T09:03:59.481 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5200 IOs, 1945916416 bytes 2024-09-14T09:03:59.664 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5300 IOs, 2009048064 bytes 2024-09-14T09:03:59.842 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5400 IOs, 2031793152 bytes 2024-09-14T09:04:00.024 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5500 IOs, 2075830272 bytes 2024-09-14T09:04:00.405 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5600 IOs, 2136897536 bytes 2024-09-14T09:04:00.603 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5700 IOs, 2183612416 bytes 2024-09-14T09:04:00.762 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5800 IOs, 2213728256 bytes 2024-09-14T09:04:00.924 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5900 IOs, 2249340928 bytes 2024-09-14T09:04:01.045 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6000 IOs, 2269946880 bytes 2024-09-14T09:04:01.250 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6100 IOs, 2329856000 bytes 2024-09-14T09:04:01.450 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6200 IOs, 2371825664 bytes 2024-09-14T09:04:01.829 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6300 IOs, 2426153984 bytes 2024-09-14T09:04:02.010 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6400 IOs, 2467044352 bytes 2024-09-14T09:04:02.204 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6500 IOs, 2524377088 bytes 2024-09-14T09:04:02.435 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6600 IOs, 2591805440 bytes 2024-09-14T09:04:02.668 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6700 IOs, 2644973568 bytes 2024-09-14T09:04:02.740 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ error_code=124 2024-09-14T09:04:02.740 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set -e 2024-09-14T09:04:02.740 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T09:04:02.740 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:04:02.740 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_present cluster1 mirror image_2 present 2024-09-14T09:04:02.740 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:04:02.740 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:04:02.741 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_2 2024-09-14T09:04:02.741 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=present 2024-09-14T09:04:02.741 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:04:02.741 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T09:04:02.741 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:04:02.741 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_2 2024-09-14T09:04:02.741 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:04:02.741 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:04:02.741 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_2 2024-09-14T09:04:02.742 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_2 2024-09-14T09:04:02.742 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:04:02.789 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image_id=153973283e38 2024-09-14T09:04:02.789 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T09:04:02.789 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.1 2024-09-14T09:04:02.890 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_present cluster1 mirror image_2 present 153973283e38 2024-09-14T09:04:02.891 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:04:02.891 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:04:02.891 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_2 2024-09-14T09:04:02.891 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=present 2024-09-14T09:04:02.891 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id=153973283e38 2024-09-14T09:04:02.891 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=deleted 2024-09-14T09:04:02.891 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_image_id 2024-09-14T09:04:02.891 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_2 2024-09-14T09:04:02.891 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:04:02.891 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:04:02.891 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_2 2024-09-14T09:04:02.891 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_2 2024-09-14T09:04:02.892 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:04:02.940 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_image_id=153973283e38 2024-09-14T09:04:02.940 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n 153973283e38 2024-09-14T09:04:02.940 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -z 153973283e38 -o 153973283e38 = 153973283e38 2024-09-14T09:04:02.940 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_state=present 2024-09-14T09:04:02.941 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test present = present 2024-09-14T09:04:02.941 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:04:02.941 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ disable_mirror cluster2 mirror image_2 2024-09-14T09:04:02.941 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:04:02.941 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:04:02.941 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_2 2024-09-14T09:04:02.941 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_2 2024-09-14T09:04:03.698 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi164.stderr:2024-09-14T09:04:03.696+0000 7f0bfe298640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x560c3df77000 scan_remote_mirror_snapshots: failed to locate remote start snapshot: snap_id=186 2024-09-14T09:04:03.698 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi164.stderr:2024-09-14T09:04:03.696+0000 7f0bfe298640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x560c3df77000 scan_remote_mirror_snapshots: split-brain detected: failed to find matching non-primary snapshot in remote image: local_snap_id_start=188, local_snap_ns=[mirror state=non-primary, complete=1, mirror_peer_uuids=, primary_mirror_uuid=9ab74346-7215-4f9b-943c-26e4f257da03, primary_snap_id=ba, last_copied_object_number=128, snap_seqs={186=18446744073709551614}] 2024-09-14T09:04:08.697 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Mirroring disabled 2024-09-14T09:04:08.706 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_present cluster1 mirror image_2 deleted 2024-09-14T09:04:08.706 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:04:08.706 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:04:08.706 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_2 2024-09-14T09:04:08.706 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=deleted 2024-09-14T09:04:08.706 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:04:08.706 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T09:04:08.706 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:04:08.706 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_2 2024-09-14T09:04:08.707 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:04:08.707 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:04:08.707 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_2 2024-09-14T09:04:08.707 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_2 2024-09-14T09:04:08.707 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:04:08.748 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_2: (2) No such file or directory 2024-09-14T09:04:08.752 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image_id= 2024-09-14T09:04:08.752 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T09:04:08.752 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.1 2024-09-14T09:04:08.854 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_present cluster1 mirror image_2 deleted '' 2024-09-14T09:04:08.854 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:04:08.854 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:04:08.854 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_2 2024-09-14T09:04:08.854 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=deleted 2024-09-14T09:04:08.855 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:04:08.855 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=deleted 2024-09-14T09:04:08.855 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_image_id 2024-09-14T09:04:08.855 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_2 2024-09-14T09:04:08.855 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:04:08.855 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:04:08.855 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_2 2024-09-14T09:04:08.855 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_2 2024-09-14T09:04:08.855 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:04:09.094 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_2: (2) No such file or directory 2024-09-14T09:04:09.099 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_image_id= 2024-09-14T09:04:09.099 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:04:09.099 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test deleted = deleted 2024-09-14T09:04:09.099 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:04:09.099 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ purge_snapshots cluster2 mirror image_2 2024-09-14T09:04:09.099 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:04:09.099 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:04:09.099 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_2 2024-09-14T09:04:09.099 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap purge mirror/image_2 2024-09-14T09:04:09.150 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image_retry cluster2 mirror image_2 2024-09-14T09:04:09.150 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:04:09.150 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:04:09.151 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_2 2024-09-14T09:04:09.151 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:04:09.151 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0 2024-09-14T09:04:09.151 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_2 2024-09-14T09:04:09.152 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:04:09.152 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:04:09.152 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_2 2024-09-14T09:04:09.152 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_2 2024-09-14T09:04:09.200 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_2 2024-09-14T09:04:09.245 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 0% complete...failed. 2024-09-14T09:04:09.246 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error: image still has watchers 2024-09-14T09:04:09.246 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout. 2024-09-14T09:04:09.248 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:04:09.249 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T09:04:10.250 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_2 2024-09-14T09:04:10.250 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:04:10.250 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:04:10.250 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_2 2024-09-14T09:04:10.250 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_2 2024-09-14T09:04:10.298 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_2 2024-09-14T09:04:10.344 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 0% complete...failed. 2024-09-14T09:04:10.345 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error: image still has watchers 2024-09-14T09:04:10.345 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout. 2024-09-14T09:04:10.347 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:04:10.348 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 2 2024-09-14T09:04:12.349 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_2 2024-09-14T09:04:12.349 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:04:12.349 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:04:12.350 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_2 2024-09-14T09:04:12.350 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_2 2024-09-14T09:04:12.404 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_2 2024-09-14T09:04:12.453 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 0% complete...failed. 2024-09-14T09:04:12.453 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error: image still has watchers 2024-09-14T09:04:12.453 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout. 2024-09-14T09:04:12.456 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:04:12.457 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 4 2024-09-14T09:04:16.458 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_2 2024-09-14T09:04:16.458 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:04:16.458 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:04:16.458 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_2 2024-09-14T09:04:16.459 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_2 2024-09-14T09:04:16.512 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_2 2024-09-14T09:04:16.561 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 0% complete...failed. 2024-09-14T09:04:16.561 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error: image still has watchers 2024-09-14T09:04:16.561 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout. 2024-09-14T09:04:16.564 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:04:16.565 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 8 2024-09-14T09:04:24.566 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_2 2024-09-14T09:04:24.566 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:04:24.566 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:04:24.566 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_2 2024-09-14T09:04:24.566 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_2 2024-09-14T09:04:24.620 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_2 2024-09-14T09:04:24.669 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 0% complete...failed. 2024-09-14T09:04:24.669 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error: image still has watchers 2024-09-14T09:04:24.669 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout. 2024-09-14T09:04:24.672 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:04:24.672 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 16 2024-09-14T09:04:40.673 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_2 2024-09-14T09:04:40.674 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:04:40.674 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:04:40.674 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_2 2024-09-14T09:04:40.674 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_2 2024-09-14T09:04:40.730 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_2 2024-09-14T09:04:40.941 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-14T09:04:40.945 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:04:40.945 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 10` 2024-09-14T09:04:40.945 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_3 2024-09-14T09:04:40.945 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image cluster2 mirror image_3 512M 2024-09-14T09:04:40.946 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:04:40.946 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T09:04:40.946 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:04:40.946 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T09:04:40.946 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_3 2024-09-14T09:04:40.946 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T09:04:40.946 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local size=128 2024-09-14T09:04:40.946 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n 512M ']' 2024-09-14T09:04:40.946 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ size=512M 2024-09-14T09:04:40.946 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T09:04:40.946 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering mirror/image_3 2024-09-14T09:04:40.995 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ enable_mirror cluster2 mirror image_3 2024-09-14T09:04:40.995 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:04:40.995 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:04:40.995 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_3 2024-09-14T09:04:40.995 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T09:04:40.995 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_3 snapshot 2024-09-14T09:04:41.758 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Mirroring enabled 2024-09-14T09:04:41.764 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 info mirror/image_3 2024-09-14T09:04:41.805 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_3': 2024-09-14T09:04:41.805 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 512 MiB in 128 objects 2024-09-14T09:04:41.805 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T09:04:41.805 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 1 2024-09-14T09:04:41.805 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 19464fd51ca4 2024-09-14T09:04:41.805 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.19464fd51ca4 2024-09-14T09:04:41.805 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T09:04:41.805 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering 2024-09-14T09:04:41.805 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T09:04:41.806 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T09:04:41.806 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 09:04:40 2024 2024-09-14T09:04:41.806 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 09:04:40 2024 2024-09-14T09:04:41.806 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 09:04:40 2024 2024-09-14T09:04:41.806 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T09:04:41.806 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T09:04:41.806 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: e9a39b0a-9a56-4d3f-bcfe-8d74663b2c35 2024-09-14T09:04:41.806 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: true 2024-09-14T09:04:41.808 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ stress_write_image cluster2 mirror image_3 2024-09-14T09:04:41.808 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:04:41.808 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:04:41.808 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_3 2024-09-14T09:04:41.808 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T09:04:41.810 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local duration=14 2024-09-14T09:04:41.810 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set +e 2024-09-14T09:04:41.810 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ timeout 14s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_3 --debug-rbd=20 --debug-journaler=20 2024-09-14T09:04:42.080 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:100 IOs, 64929792 bytes 2024-09-14T09:04:42.201 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:200 IOs, 81346560 bytes 2024-09-14T09:04:42.400 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:300 IOs, 140888064 bytes 2024-09-14T09:04:42.599 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:400 IOs, 185402368 bytes 2024-09-14T09:04:42.677 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:500 IOs, 186010624 bytes 2024-09-14T09:04:42.757 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:600 IOs, 187380736 bytes 2024-09-14T09:04:43.031 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:700 IOs, 274274304 bytes 2024-09-14T09:04:43.143 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:800 IOs, 288583680 bytes 2024-09-14T09:04:43.289 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:900 IOs, 319268864 bytes 2024-09-14T09:04:43.411 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1000 IOs, 338385920 bytes 2024-09-14T09:04:43.661 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1100 IOs, 410961920 bytes 2024-09-14T09:04:43.816 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1200 IOs, 441933824 bytes 2024-09-14T09:04:44.050 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1300 IOs, 529436672 bytes 2024-09-14T09:04:44.271 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1400 IOs, 567753728 bytes 2024-09-14T09:04:44.409 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1500 IOs, 598532096 bytes 2024-09-14T09:04:44.609 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1600 IOs, 642475008 bytes 2024-09-14T09:04:44.837 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1700 IOs, 716260352 bytes 2024-09-14T09:04:45.057 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1800 IOs, 773937152 bytes 2024-09-14T09:04:45.214 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1900 IOs, 798155776 bytes 2024-09-14T09:04:45.383 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2000 IOs, 833039360 bytes 2024-09-14T09:04:45.534 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2100 IOs, 868172800 bytes 2024-09-14T09:04:45.632 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2200 IOs, 886475776 bytes 2024-09-14T09:04:45.884 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2300 IOs, 944994304 bytes 2024-09-14T09:04:45.999 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2400 IOs, 977890304 bytes 2024-09-14T09:04:46.260 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2500 IOs, 1047953408 bytes 2024-09-14T09:04:46.413 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2600 IOs, 1085826048 bytes 2024-09-14T09:04:46.651 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2700 IOs, 1142171648 bytes 2024-09-14T09:04:46.833 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2800 IOs, 1187575808 bytes 2024-09-14T09:04:46.967 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2900 IOs, 1207035904 bytes 2024-09-14T09:04:47.202 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3000 IOs, 1270721536 bytes 2024-09-14T09:04:47.329 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3100 IOs, 1289148416 bytes 2024-09-14T09:04:47.512 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3200 IOs, 1350352896 bytes 2024-09-14T09:04:47.740 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3300 IOs, 1403488256 bytes 2024-09-14T09:04:47.874 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3400 IOs, 1427683328 bytes 2024-09-14T09:04:48.173 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3500 IOs, 1518999552 bytes 2024-09-14T09:04:48.333 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3600 IOs, 1565772800 bytes 2024-09-14T09:04:48.493 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3700 IOs, 1587239936 bytes 2024-09-14T09:04:48.585 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3800 IOs, 1593174016 bytes 2024-09-14T09:04:48.729 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3900 IOs, 1636200448 bytes 2024-09-14T09:04:49.016 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4000 IOs, 1714501632 bytes 2024-09-14T09:04:49.210 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4100 IOs, 1776969728 bytes 2024-09-14T09:04:49.436 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4200 IOs, 1820178432 bytes 2024-09-14T09:04:49.535 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4300 IOs, 1830289408 bytes 2024-09-14T09:04:49.818 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4400 IOs, 1918308352 bytes 2024-09-14T09:04:49.921 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4500 IOs, 1934005248 bytes 2024-09-14T09:04:50.087 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4600 IOs, 1969878016 bytes 2024-09-14T09:04:50.203 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4700 IOs, 1986343936 bytes 2024-09-14T09:04:50.367 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4800 IOs, 2048019456 bytes 2024-09-14T09:04:50.573 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4900 IOs, 2069472256 bytes 2024-09-14T09:04:50.796 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5000 IOs, 2145494016 bytes 2024-09-14T09:04:51.027 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5100 IOs, 2191342592 bytes 2024-09-14T09:04:51.242 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5200 IOs, 2260097024 bytes 2024-09-14T09:04:51.555 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5300 IOs, 2348649472 bytes 2024-09-14T09:04:51.784 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5400 IOs, 2408745984 bytes 2024-09-14T09:04:52.018 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5500 IOs, 2477647872 bytes 2024-09-14T09:04:52.178 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5600 IOs, 2509842432 bytes 2024-09-14T09:04:52.334 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5700 IOs, 2543128576 bytes 2024-09-14T09:04:52.553 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5800 IOs, 2600768512 bytes 2024-09-14T09:04:52.790 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5900 IOs, 2663358464 bytes 2024-09-14T09:04:52.937 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6000 IOs, 2689042432 bytes 2024-09-14T09:04:53.093 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6100 IOs, 2758496256 bytes 2024-09-14T09:04:53.305 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6200 IOs, 2785772544 bytes 2024-09-14T09:04:53.466 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6300 IOs, 2811806720 bytes 2024-09-14T09:04:53.731 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6400 IOs, 2896219136 bytes 2024-09-14T09:04:53.886 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6500 IOs, 2921454592 bytes 2024-09-14T09:04:54.123 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6600 IOs, 2988503040 bytes 2024-09-14T09:04:54.324 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6700 IOs, 3040036864 bytes 2024-09-14T09:04:54.499 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6800 IOs, 3079473152 bytes 2024-09-14T09:04:54.681 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6900 IOs, 3127091200 bytes 2024-09-14T09:04:54.800 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:7000 IOs, 3139842048 bytes 2024-09-14T09:04:54.917 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:7100 IOs, 3155430400 bytes 2024-09-14T09:04:55.062 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:7200 IOs, 3184502784 bytes 2024-09-14T09:04:55.284 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:7300 IOs, 3261173760 bytes 2024-09-14T09:04:55.494 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:7400 IOs, 3296907264 bytes 2024-09-14T09:04:55.807 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:7500 IOs, 3412073472 bytes 2024-09-14T09:04:55.817 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ error_code=124 2024-09-14T09:04:55.817 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set -e 2024-09-14T09:04:55.817 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T09:04:55.817 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:04:55.817 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_present cluster1 mirror image_3 present 2024-09-14T09:04:55.817 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:04:55.818 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:04:55.818 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_3 2024-09-14T09:04:55.818 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=present 2024-09-14T09:04:55.818 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:04:55.818 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T09:04:55.818 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:04:55.818 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_3 2024-09-14T09:04:55.818 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:04:55.818 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:04:55.819 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_3 2024-09-14T09:04:55.819 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_3 2024-09-14T09:04:55.819 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:04:55.874 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image_id=15394aa93fce 2024-09-14T09:04:55.874 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T09:04:55.874 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.1 2024-09-14T09:04:55.976 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_present cluster1 mirror image_3 present 15394aa93fce 2024-09-14T09:04:55.976 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:04:55.976 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:04:55.976 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_3 2024-09-14T09:04:55.976 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=present 2024-09-14T09:04:55.976 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id=15394aa93fce 2024-09-14T09:04:55.976 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=deleted 2024-09-14T09:04:55.977 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_image_id 2024-09-14T09:04:55.977 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_3 2024-09-14T09:04:55.977 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:04:55.977 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:04:55.977 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_3 2024-09-14T09:04:55.977 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_3 2024-09-14T09:04:55.977 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:04:56.023 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_image_id=15394aa93fce 2024-09-14T09:04:56.023 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n 15394aa93fce 2024-09-14T09:04:56.024 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -z 15394aa93fce -o 15394aa93fce = 15394aa93fce 2024-09-14T09:04:56.024 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_state=present 2024-09-14T09:04:56.024 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test present = present 2024-09-14T09:04:56.024 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:04:56.024 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ disable_mirror cluster2 mirror image_3 2024-09-14T09:04:56.024 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:04:56.024 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:04:56.024 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_3 2024-09-14T09:04:56.024 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_3 2024-09-14T09:04:56.361 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi164.stderr:2024-09-14T09:04:56.359+0000 7f0bfea99640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x560c3bb5cc00 scan_remote_mirror_snapshots: failed to locate remote start snapshot: snap_id=187 2024-09-14T09:04:56.361 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi164.stderr:2024-09-14T09:04:56.359+0000 7f0bfea99640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x560c3bb5cc00 scan_remote_mirror_snapshots: split-brain detected: failed to find matching non-primary snapshot in remote image: local_snap_id_start=189, local_snap_ns=[mirror state=non-primary, complete=1, mirror_peer_uuids=, primary_mirror_uuid=9ab74346-7215-4f9b-943c-26e4f257da03, primary_snap_id=bb, last_copied_object_number=128, snap_seqs={187=18446744073709551614}] 2024-09-14T09:05:01.359 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Mirroring disabled 2024-09-14T09:05:01.367 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_present cluster1 mirror image_3 deleted 2024-09-14T09:05:01.367 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:05:01.367 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:05:01.367 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_3 2024-09-14T09:05:01.367 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=deleted 2024-09-14T09:05:01.367 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:05:01.367 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T09:05:01.368 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:05:01.368 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_3 2024-09-14T09:05:01.368 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:05:01.368 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:05:01.368 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_3 2024-09-14T09:05:01.369 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_3 2024-09-14T09:05:01.369 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:05:01.389 INFO:tasks.ceph.cluster1.osd.2.smithi135.stderr:2024-09-14T09:05:01.388+0000 7f317814f640 -1 reset not still connected to 0x559cc6361e10 2024-09-14T09:05:01.613 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_3: (2) No such file or directory 2024-09-14T09:05:01.616 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image_id= 2024-09-14T09:05:01.616 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T09:05:01.617 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.1 2024-09-14T09:05:01.721 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_present cluster1 mirror image_3 deleted '' 2024-09-14T09:05:01.721 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:05:01.721 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:05:01.721 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_3 2024-09-14T09:05:01.722 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=deleted 2024-09-14T09:05:01.722 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:05:01.722 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=deleted 2024-09-14T09:05:01.722 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_image_id 2024-09-14T09:05:01.722 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_3 2024-09-14T09:05:01.722 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:05:01.722 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:05:01.722 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_3 2024-09-14T09:05:01.722 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_3 2024-09-14T09:05:01.722 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:05:01.760 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_3: (2) No such file or directory 2024-09-14T09:05:01.764 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_image_id= 2024-09-14T09:05:01.764 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:05:01.764 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test deleted = deleted 2024-09-14T09:05:01.764 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:05:01.764 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ purge_snapshots cluster2 mirror image_3 2024-09-14T09:05:01.764 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:05:01.765 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:05:01.765 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_3 2024-09-14T09:05:01.765 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap purge mirror/image_3 2024-09-14T09:05:01.815 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image_retry cluster2 mirror image_3 2024-09-14T09:05:01.815 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:05:01.815 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:05:01.815 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_3 2024-09-14T09:05:01.815 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:05:01.815 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0 2024-09-14T09:05:01.816 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_3 2024-09-14T09:05:01.816 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:05:01.816 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:05:01.816 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_3 2024-09-14T09:05:01.816 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_3 2024-09-14T09:05:01.863 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_3 2024-09-14T09:05:01.908 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 0% complete...failed. 2024-09-14T09:05:01.909 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error: image still has watchers 2024-09-14T09:05:01.909 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout. 2024-09-14T09:05:01.912 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:05:01.912 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T09:05:02.193 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.9910426623205444) > weight (0.25) 2024-09-14T09:05:02.193 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.2703525734718628) > weight (0.25) 2024-09-14T09:05:02.193 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.3458753639008524) > weight (0.25) 2024-09-14T09:05:02.194 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.3 with skip (0.7355962789841592) > weight (0.25) 2024-09-14T09:05:02.194 INFO:tasks.rbd_mirror_thrash:waiting for 106 secs before thrashing 2024-09-14T09:05:02.913 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_3 2024-09-14T09:05:02.913 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:05:02.913 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:05:02.913 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_3 2024-09-14T09:05:02.913 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_3 2024-09-14T09:05:02.966 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_3 2024-09-14T09:05:03.014 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 0% complete...failed. 2024-09-14T09:05:03.014 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error: image still has watchers 2024-09-14T09:05:03.014 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout. 2024-09-14T09:05:03.017 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:05:03.017 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 2 2024-09-14T09:05:05.019 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_3 2024-09-14T09:05:05.019 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:05:05.019 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:05:05.019 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_3 2024-09-14T09:05:05.019 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_3 2024-09-14T09:05:05.074 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_3 2024-09-14T09:05:05.123 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 0% complete...failed. 2024-09-14T09:05:05.123 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error: image still has watchers 2024-09-14T09:05:05.123 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout. 2024-09-14T09:05:05.127 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:05:05.127 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 4 2024-09-14T09:05:09.128 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_3 2024-09-14T09:05:09.129 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:05:09.129 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:05:09.129 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_3 2024-09-14T09:05:09.129 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_3 2024-09-14T09:05:09.183 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_3 2024-09-14T09:05:09.232 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 0% complete...failed. 2024-09-14T09:05:09.232 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error: image still has watchers 2024-09-14T09:05:09.232 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout. 2024-09-14T09:05:09.236 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:05:09.236 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 8 2024-09-14T09:05:17.237 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_3 2024-09-14T09:05:17.237 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:05:17.237 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:05:17.238 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_3 2024-09-14T09:05:17.238 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_3 2024-09-14T09:05:17.291 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_3 2024-09-14T09:05:17.341 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 0% complete...failed. 2024-09-14T09:05:17.341 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error: image still has watchers 2024-09-14T09:05:17.341 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout. 2024-09-14T09:05:17.344 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:05:17.344 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 16 2024-09-14T09:05:33.346 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_3 2024-09-14T09:05:33.346 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:05:33.347 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:05:33.347 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_3 2024-09-14T09:05:33.347 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_3 2024-09-14T09:05:33.401 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_3 2024-09-14T09:05:33.619 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-14T09:05:33.621 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:05:33.622 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 10` 2024-09-14T09:05:33.622 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_4 2024-09-14T09:05:33.622 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image cluster2 mirror image_4 512M 2024-09-14T09:05:33.622 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:05:33.622 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T09:05:33.622 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:05:33.622 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T09:05:33.622 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_4 2024-09-14T09:05:33.622 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T09:05:33.622 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local size=128 2024-09-14T09:05:33.623 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n 512M ']' 2024-09-14T09:05:33.623 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ size=512M 2024-09-14T09:05:33.623 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T09:05:33.623 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering mirror/image_4 2024-09-14T09:05:33.671 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ enable_mirror cluster2 mirror image_4 2024-09-14T09:05:33.671 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:05:33.671 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:05:33.671 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_4 2024-09-14T09:05:33.671 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T09:05:33.671 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_4 snapshot 2024-09-14T09:05:34.664 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Mirroring enabled 2024-09-14T09:05:34.671 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 info mirror/image_4 2024-09-14T09:05:34.712 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_4': 2024-09-14T09:05:34.712 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 512 MiB in 128 objects 2024-09-14T09:05:34.712 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T09:05:34.713 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 1 2024-09-14T09:05:34.713 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 197cb9379871 2024-09-14T09:05:34.713 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.197cb9379871 2024-09-14T09:05:34.713 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T09:05:34.713 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering 2024-09-14T09:05:34.713 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T09:05:34.713 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T09:05:34.713 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 09:05:33 2024 2024-09-14T09:05:34.713 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 09:05:33 2024 2024-09-14T09:05:34.713 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 09:05:33 2024 2024-09-14T09:05:34.713 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T09:05:34.713 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T09:05:34.713 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: d9f42c69-a437-429b-a656-f31c3a732b4d 2024-09-14T09:05:34.714 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: true 2024-09-14T09:05:34.716 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ stress_write_image cluster2 mirror image_4 2024-09-14T09:05:34.716 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:05:34.716 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:05:34.716 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_4 2024-09-14T09:05:34.716 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T09:05:34.718 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local duration=9 2024-09-14T09:05:34.718 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set +e 2024-09-14T09:05:34.718 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ timeout 9s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_4 --debug-rbd=20 --debug-journaler=20 2024-09-14T09:05:34.926 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:100 IOs, 36045824 bytes 2024-09-14T09:05:35.009 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:200 IOs, 51976192 bytes 2024-09-14T09:05:35.117 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:300 IOs, 52614144 bytes 2024-09-14T09:05:35.271 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:400 IOs, 86522880 bytes 2024-09-14T09:05:35.386 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:500 IOs, 111502336 bytes 2024-09-14T09:05:35.522 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:600 IOs, 126400512 bytes 2024-09-14T09:05:35.700 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:700 IOs, 168215552 bytes 2024-09-14T09:05:35.849 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:800 IOs, 201165824 bytes 2024-09-14T09:05:35.993 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:900 IOs, 223356928 bytes 2024-09-14T09:05:36.232 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1000 IOs, 292572160 bytes 2024-09-14T09:05:36.373 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1100 IOs, 324865024 bytes 2024-09-14T09:05:36.559 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1200 IOs, 378169344 bytes 2024-09-14T09:05:36.737 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1300 IOs, 410346496 bytes 2024-09-14T09:05:36.897 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1400 IOs, 463358976 bytes 2024-09-14T09:05:37.062 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1500 IOs, 484488192 bytes 2024-09-14T09:05:37.200 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1600 IOs, 510196736 bytes 2024-09-14T09:05:37.417 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1700 IOs, 565221376 bytes 2024-09-14T09:05:37.642 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1800 IOs, 625930240 bytes 2024-09-14T09:05:37.822 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1900 IOs, 674369536 bytes 2024-09-14T09:05:38.032 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2000 IOs, 749481984 bytes 2024-09-14T09:05:38.170 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2100 IOs, 750110720 bytes 2024-09-14T09:05:38.316 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2200 IOs, 777857024 bytes 2024-09-14T09:05:38.501 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2300 IOs, 823858176 bytes 2024-09-14T09:05:38.651 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2400 IOs, 849130496 bytes 2024-09-14T09:05:38.780 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2500 IOs, 871172096 bytes 2024-09-14T09:05:39.036 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2600 IOs, 948837376 bytes 2024-09-14T09:05:39.175 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2700 IOs, 972824576 bytes 2024-09-14T09:05:39.377 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2800 IOs, 1049771008 bytes 2024-09-14T09:05:39.579 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2900 IOs, 1076327424 bytes 2024-09-14T09:05:39.705 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3000 IOs, 1100573696 bytes 2024-09-14T09:05:39.948 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3100 IOs, 1168014336 bytes 2024-09-14T09:05:40.062 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.7344605978423108) > weight (0.25) 2024-09-14T09:05:40.062 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.1 2024-09-14T09:05:40.063 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Sent signal 15 2024-09-14T09:05:40.063 INFO:tasks.rbd_mirror_thrash:waiting for 9 secs before reviving daemons 2024-09-14T09:05:40.064 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi164.stderr:2024-09-14T09:05:40.064+0000 7f92344c6640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 (PID: 68954) UID: 1000 2024-09-14T09:05:40.199 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3200 IOs, 1236224000 bytes 2024-09-14T09:05:40.434 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3300 IOs, 1305888768 bytes 2024-09-14T09:05:40.629 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3400 IOs, 1350547456 bytes 2024-09-14T09:05:40.736 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3500 IOs, 1364391936 bytes 2024-09-14T09:05:40.889 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3600 IOs, 1398664192 bytes 2024-09-14T09:05:41.057 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3700 IOs, 1432685568 bytes 2024-09-14T09:05:41.215 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3800 IOs, 1466861568 bytes 2024-09-14T09:05:41.405 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3900 IOs, 1514153984 bytes 2024-09-14T09:05:41.562 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4000 IOs, 1553319936 bytes 2024-09-14T09:05:41.852 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4100 IOs, 1658659840 bytes 2024-09-14T09:05:42.042 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4200 IOs, 1686596608 bytes 2024-09-14T09:05:42.133 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4300 IOs, 1693720576 bytes 2024-09-14T09:05:42.288 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4400 IOs, 1725824000 bytes 2024-09-14T09:05:42.580 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4500 IOs, 1814079488 bytes 2024-09-14T09:05:42.686 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4600 IOs, 1823894528 bytes 2024-09-14T09:05:42.846 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4700 IOs, 1857139712 bytes 2024-09-14T09:05:43.175 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4800 IOs, 1959841792 bytes 2024-09-14T09:05:43.402 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4900 IOs, 2031838208 bytes 2024-09-14T09:05:43.606 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5000 IOs, 2078614528 bytes 2024-09-14T09:05:43.725 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ error_code=124 2024-09-14T09:05:43.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set -e 2024-09-14T09:05:43.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T09:05:43.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:05:43.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_present cluster1 mirror image_4 present 2024-09-14T09:05:43.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:05:43.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:05:43.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_4 2024-09-14T09:05:43.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=present 2024-09-14T09:05:43.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:05:43.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T09:05:43.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:05:43.727 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_4 2024-09-14T09:05:43.727 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:05:43.727 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:05:43.727 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_4 2024-09-14T09:05:43.727 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_4 2024-09-14T09:05:43.727 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:05:43.771 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image_id=15399cced1e6 2024-09-14T09:05:43.771 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T09:05:43.771 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.1 2024-09-14T09:05:43.873 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_present cluster1 mirror image_4 present 15399cced1e6 2024-09-14T09:05:43.873 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:05:43.873 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:05:43.873 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_4 2024-09-14T09:05:43.873 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=present 2024-09-14T09:05:43.873 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id=15399cced1e6 2024-09-14T09:05:43.873 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=deleted 2024-09-14T09:05:43.874 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_image_id 2024-09-14T09:05:43.874 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_4 2024-09-14T09:05:43.874 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:05:43.874 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:05:43.874 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_4 2024-09-14T09:05:43.874 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_4 2024-09-14T09:05:43.875 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:05:43.917 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_image_id=15399cced1e6 2024-09-14T09:05:43.917 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n 15399cced1e6 2024-09-14T09:05:43.917 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -z 15399cced1e6 -o 15399cced1e6 = 15399cced1e6 2024-09-14T09:05:43.917 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_state=present 2024-09-14T09:05:43.917 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test present = present 2024-09-14T09:05:43.917 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:05:43.918 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ disable_mirror cluster2 mirror image_4 2024-09-14T09:05:43.918 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:05:43.918 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:05:43.918 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_4 2024-09-14T09:05:43.918 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_4 2024-09-14T09:05:44.677 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi164.stderr:2024-09-14T09:05:44.677+0000 7f0bfe298640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x560c3b6e3000 scan_remote_mirror_snapshots: failed to locate remote start snapshot: snap_id=188 2024-09-14T09:05:44.677 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi164.stderr:2024-09-14T09:05:44.677+0000 7f0bfe298640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x560c3b6e3000 scan_remote_mirror_snapshots: split-brain detected: failed to find matching non-primary snapshot in remote image: local_snap_id_start=190, local_snap_ns=[mirror state=non-primary, complete=1, mirror_peer_uuids=, primary_mirror_uuid=9ab74346-7215-4f9b-943c-26e4f257da03, primary_snap_id=bc, last_copied_object_number=128, snap_seqs={188=18446744073709551614}] 2024-09-14T09:05:46.316 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi164.stderr:2024-09-14T09:05:46.316+0000 7f3754609640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55c94be28820 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-14T09:05:46.316 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi164.stderr:2024-09-14T09:05:46.316+0000 7f375ae16640 -1 rbd::mirror::ImageReplayer: 0x55c94ba4cc80 [3/d9f42c69-a437-429b-a656-f31c3a732b4d] operator(): start failed: (85) Interrupted system call should be restarted 2024-09-14T09:05:49.064 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.1 2024-09-14T09:05:49.065 INFO:teuthology.orchestra.run:waiting for 600 2024-09-14T09:05:49.065 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.1 2024-09-14T09:05:49.065 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Restarting daemon 2024-09-14T09:05:49.065 DEBUG:teuthology.orchestra.run.smithi164:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 2024-09-14T09:05:49.069 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Started 2024-09-14T09:05:49.069 INFO:tasks.rbd_mirror_thrash:waiting for 75 secs before thrashing 2024-09-14T09:05:49.678 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Mirroring disabled 2024-09-14T09:05:49.686 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_present cluster1 mirror image_4 deleted 2024-09-14T09:05:49.686 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:05:49.686 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:05:49.686 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_4 2024-09-14T09:05:49.686 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=deleted 2024-09-14T09:05:49.686 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:05:49.686 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T09:05:49.686 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:05:49.687 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_4 2024-09-14T09:05:49.687 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:05:49.687 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:05:49.687 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_4 2024-09-14T09:05:49.687 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_4 2024-09-14T09:05:49.687 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:05:49.722 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_4: (2) No such file or directory 2024-09-14T09:05:49.725 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image_id= 2024-09-14T09:05:49.725 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T09:05:49.725 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.1 2024-09-14T09:05:49.827 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_present cluster1 mirror image_4 deleted '' 2024-09-14T09:05:49.827 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:05:49.827 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:05:49.827 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_4 2024-09-14T09:05:49.828 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=deleted 2024-09-14T09:05:49.828 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:05:49.828 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=deleted 2024-09-14T09:05:49.828 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_image_id 2024-09-14T09:05:49.828 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_4 2024-09-14T09:05:49.828 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:05:49.828 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:05:49.828 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_4 2024-09-14T09:05:49.828 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_4 2024-09-14T09:05:49.828 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:05:49.862 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_4: (2) No such file or directory 2024-09-14T09:05:49.866 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_image_id= 2024-09-14T09:05:49.866 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:05:49.866 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test deleted = deleted 2024-09-14T09:05:49.866 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:05:49.866 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ purge_snapshots cluster2 mirror image_4 2024-09-14T09:05:49.866 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:05:49.866 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:05:49.866 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_4 2024-09-14T09:05:49.866 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap purge mirror/image_4 2024-09-14T09:05:49.912 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image_retry cluster2 mirror image_4 2024-09-14T09:05:49.920 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:05:49.920 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:05:49.920 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_4 2024-09-14T09:05:49.921 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:05:49.921 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0 2024-09-14T09:05:49.921 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_4 2024-09-14T09:05:49.921 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:05:49.921 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:05:49.922 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_4 2024-09-14T09:05:49.922 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_4 2024-09-14T09:05:49.960 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_4 2024-09-14T09:05:50.005 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 0% complete...failed. 2024-09-14T09:05:50.005 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error: image still has watchers 2024-09-14T09:05:50.005 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout. 2024-09-14T09:05:50.008 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:05:50.008 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T09:05:51.012 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_4 2024-09-14T09:05:51.012 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:05:51.012 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:05:51.012 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_4 2024-09-14T09:05:51.012 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_4 2024-09-14T09:05:51.065 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_4 2024-09-14T09:05:51.114 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 0% complete...failed. 2024-09-14T09:05:51.114 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error: image still has watchers 2024-09-14T09:05:51.114 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout. 2024-09-14T09:05:51.118 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:05:51.118 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 2 2024-09-14T09:05:53.120 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_4 2024-09-14T09:05:53.120 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:05:53.120 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:05:53.120 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_4 2024-09-14T09:05:53.121 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_4 2024-09-14T09:05:53.172 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_4 2024-09-14T09:05:53.221 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 0% complete...failed. 2024-09-14T09:05:53.221 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error: image still has watchers 2024-09-14T09:05:53.221 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout. 2024-09-14T09:05:53.224 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:05:53.224 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 4 2024-09-14T09:05:57.226 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_4 2024-09-14T09:05:57.226 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:05:57.226 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:05:57.226 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_4 2024-09-14T09:05:57.226 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_4 2024-09-14T09:05:57.279 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_4 2024-09-14T09:05:57.328 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 0% complete...failed. 2024-09-14T09:05:57.328 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error: image still has watchers 2024-09-14T09:05:57.329 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout. 2024-09-14T09:05:57.332 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:05:57.332 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 8 2024-09-14T09:06:05.334 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_4 2024-09-14T09:06:05.334 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:06:05.334 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:06:05.334 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_4 2024-09-14T09:06:05.334 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_4 2024-09-14T09:06:05.387 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_4 2024-09-14T09:06:05.435 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 0% complete...failed. 2024-09-14T09:06:05.436 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error: image still has watchers 2024-09-14T09:06:05.436 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout. 2024-09-14T09:06:05.439 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:06:05.439 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 16 2024-09-14T09:06:21.441 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_4 2024-09-14T09:06:21.441 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:06:21.441 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:06:21.441 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_4 2024-09-14T09:06:21.441 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_4 2024-09-14T09:06:21.495 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_4 2024-09-14T09:06:21.714 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-14T09:06:21.718 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:06:21.718 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 10` 2024-09-14T09:06:21.718 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_5 2024-09-14T09:06:21.718 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image cluster2 mirror image_5 512M 2024-09-14T09:06:21.718 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:06:21.718 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T09:06:21.718 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:06:21.718 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T09:06:21.718 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_5 2024-09-14T09:06:21.718 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T09:06:21.718 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local size=128 2024-09-14T09:06:21.718 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n 512M ']' 2024-09-14T09:06:21.718 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ size=512M 2024-09-14T09:06:21.719 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T09:06:21.719 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering mirror/image_5 2024-09-14T09:06:21.767 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ enable_mirror cluster2 mirror image_5 2024-09-14T09:06:21.767 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:06:21.767 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:06:21.767 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_5 2024-09-14T09:06:21.767 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T09:06:21.768 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_5 snapshot 2024-09-14T09:06:22.667 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Mirroring enabled 2024-09-14T09:06:22.673 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 info mirror/image_5 2024-09-14T09:06:22.714 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_5': 2024-09-14T09:06:22.714 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 512 MiB in 128 objects 2024-09-14T09:06:22.714 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T09:06:22.715 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 1 2024-09-14T09:06:22.715 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 19bb104b4869 2024-09-14T09:06:22.715 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.19bb104b4869 2024-09-14T09:06:22.715 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T09:06:22.715 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering 2024-09-14T09:06:22.715 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T09:06:22.715 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T09:06:22.715 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 09:06:21 2024 2024-09-14T09:06:22.715 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 09:06:21 2024 2024-09-14T09:06:22.715 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 09:06:21 2024 2024-09-14T09:06:22.715 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T09:06:22.715 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T09:06:22.715 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: ae9226d8-393a-4b2a-8fd1-1d038e7e5187 2024-09-14T09:06:22.715 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: true 2024-09-14T09:06:22.718 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ stress_write_image cluster2 mirror image_5 2024-09-14T09:06:22.718 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:06:22.718 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:06:22.718 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_5 2024-09-14T09:06:22.718 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T09:06:22.720 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local duration=14 2024-09-14T09:06:22.720 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set +e 2024-09-14T09:06:22.720 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ timeout 14s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_5 --debug-rbd=20 --debug-journaler=20 2024-09-14T09:06:22.913 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:100 IOs, 25696256 bytes 2024-09-14T09:06:23.203 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:200 IOs, 116246528 bytes 2024-09-14T09:06:23.466 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:300 IOs, 173214720 bytes 2024-09-14T09:06:23.634 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:400 IOs, 202914816 bytes 2024-09-14T09:06:23.812 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:500 IOs, 242424832 bytes 2024-09-14T09:06:24.091 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:600 IOs, 321192960 bytes 2024-09-14T09:06:24.366 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:700 IOs, 396674048 bytes 2024-09-14T09:06:24.621 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:800 IOs, 459854848 bytes 2024-09-14T09:06:24.827 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:900 IOs, 506851328 bytes 2024-09-14T09:06:24.965 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1000 IOs, 527121408 bytes 2024-09-14T09:06:25.101 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1100 IOs, 546905088 bytes 2024-09-14T09:06:25.226 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1200 IOs, 563598336 bytes 2024-09-14T09:06:25.385 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1300 IOs, 595344384 bytes 2024-09-14T09:06:25.624 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1400 IOs, 635633664 bytes 2024-09-14T09:06:25.809 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1500 IOs, 675039232 bytes 2024-09-14T09:06:26.028 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1600 IOs, 719224832 bytes 2024-09-14T09:06:26.297 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1700 IOs, 785998848 bytes 2024-09-14T09:06:26.567 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1800 IOs, 855485440 bytes 2024-09-14T09:06:26.697 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1900 IOs, 875506688 bytes 2024-09-14T09:06:26.925 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2000 IOs, 934383616 bytes 2024-09-14T09:06:27.015 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2100 IOs, 937496576 bytes 2024-09-14T09:06:27.203 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2200 IOs, 975917056 bytes 2024-09-14T09:06:27.440 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2300 IOs, 1035414528 bytes 2024-09-14T09:06:27.814 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2400 IOs, 1159802880 bytes 2024-09-14T09:06:28.036 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2500 IOs, 1215936512 bytes 2024-09-14T09:06:28.277 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2600 IOs, 1264314368 bytes 2024-09-14T09:06:28.411 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2700 IOs, 1280573440 bytes 2024-09-14T09:06:28.493 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2800 IOs, 1281223680 bytes 2024-09-14T09:06:28.574 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2900 IOs, 1283610624 bytes 2024-09-14T09:06:28.803 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3000 IOs, 1354870784 bytes 2024-09-14T09:06:29.109 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3100 IOs, 1428770816 bytes 2024-09-14T09:06:29.271 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3200 IOs, 1449502720 bytes 2024-09-14T09:06:29.353 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3300 IOs, 1450106880 bytes 2024-09-14T09:06:29.596 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3400 IOs, 1509588992 bytes 2024-09-14T09:06:29.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3500 IOs, 1533191168 bytes 2024-09-14T09:06:29.888 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3600 IOs, 1566816256 bytes 2024-09-14T09:06:30.155 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3700 IOs, 1652091904 bytes 2024-09-14T09:06:30.384 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3800 IOs, 1682690048 bytes 2024-09-14T09:06:30.515 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3900 IOs, 1704349696 bytes 2024-09-14T09:06:30.733 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4000 IOs, 1755554816 bytes 2024-09-14T09:06:30.809 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4100 IOs, 1756181504 bytes 2024-09-14T09:06:31.013 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4200 IOs, 1799924736 bytes 2024-09-14T09:06:31.113 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4300 IOs, 1812934656 bytes 2024-09-14T09:06:31.397 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4400 IOs, 1888124928 bytes 2024-09-14T09:06:31.493 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4500 IOs, 1888730112 bytes 2024-09-14T09:06:31.633 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4600 IOs, 1915832320 bytes 2024-09-14T09:06:31.888 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4700 IOs, 1979987968 bytes 2024-09-14T09:06:32.039 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4800 IOs, 1990836224 bytes 2024-09-14T09:06:32.119 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4900 IOs, 1991470080 bytes 2024-09-14T09:06:32.427 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5000 IOs, 2079942656 bytes 2024-09-14T09:06:32.526 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5100 IOs, 2083097600 bytes 2024-09-14T09:06:32.668 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5200 IOs, 2109340672 bytes 2024-09-14T09:06:32.934 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5300 IOs, 2173116416 bytes 2024-09-14T09:06:33.097 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5400 IOs, 2220609536 bytes 2024-09-14T09:06:33.312 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5500 IOs, 2266252288 bytes 2024-09-14T09:06:33.614 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5600 IOs, 2332023808 bytes 2024-09-14T09:06:33.763 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5700 IOs, 2351045632 bytes 2024-09-14T09:06:33.851 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5800 IOs, 2354405376 bytes 2024-09-14T09:06:34.009 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5900 IOs, 2384411648 bytes 2024-09-14T09:06:34.231 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6000 IOs, 2444374016 bytes 2024-09-14T09:06:34.542 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6100 IOs, 2525504512 bytes 2024-09-14T09:06:34.731 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6200 IOs, 2556125184 bytes 2024-09-14T09:06:34.895 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6300 IOs, 2588547072 bytes 2024-09-14T09:06:35.037 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6400 IOs, 2609952768 bytes 2024-09-14T09:06:35.169 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6500 IOs, 2629689344 bytes 2024-09-14T09:06:35.265 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6600 IOs, 2666580992 bytes 2024-09-14T09:06:35.441 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6700 IOs, 2689434624 bytes 2024-09-14T09:06:35.623 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6800 IOs, 2711106560 bytes 2024-09-14T09:06:35.776 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6900 IOs, 2734961664 bytes 2024-09-14T09:06:35.981 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:7000 IOs, 2775885824 bytes 2024-09-14T09:06:36.118 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:7100 IOs, 2808484864 bytes 2024-09-14T09:06:36.394 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:7200 IOs, 2870863872 bytes 2024-09-14T09:06:36.623 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:7300 IOs, 2934082560 bytes 2024-09-14T09:06:36.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ error_code=124 2024-09-14T09:06:36.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set -e 2024-09-14T09:06:36.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T09:06:36.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:06:36.727 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_present cluster1 mirror image_5 present 2024-09-14T09:06:36.727 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:06:36.727 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:06:36.727 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_5 2024-09-14T09:06:36.727 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=present 2024-09-14T09:06:36.727 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:06:36.727 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T09:06:36.727 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:06:36.727 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_5 2024-09-14T09:06:36.727 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:06:36.727 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:06:36.727 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_5 2024-09-14T09:06:36.727 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_5 2024-09-14T09:06:36.727 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:06:36.788 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image_id=1539bfa18fd9 2024-09-14T09:06:36.789 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T09:06:36.789 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.1 2024-09-14T09:06:36.890 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_present cluster1 mirror image_5 present 1539bfa18fd9 2024-09-14T09:06:36.890 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:06:36.890 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:06:36.890 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_5 2024-09-14T09:06:36.890 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=present 2024-09-14T09:06:36.890 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id=1539bfa18fd9 2024-09-14T09:06:36.890 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=deleted 2024-09-14T09:06:36.890 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_image_id 2024-09-14T09:06:36.890 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_5 2024-09-14T09:06:36.891 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:06:36.891 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:06:36.891 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_5 2024-09-14T09:06:36.891 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_5 2024-09-14T09:06:36.891 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:06:36.938 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_image_id=1539bfa18fd9 2024-09-14T09:06:36.938 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n 1539bfa18fd9 2024-09-14T09:06:36.938 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -z 1539bfa18fd9 -o 1539bfa18fd9 = 1539bfa18fd9 2024-09-14T09:06:36.938 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_state=present 2024-09-14T09:06:36.938 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test present = present 2024-09-14T09:06:36.938 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:06:36.938 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ disable_mirror cluster2 mirror image_5 2024-09-14T09:06:36.938 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:06:36.939 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:06:36.939 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_5 2024-09-14T09:06:36.939 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_5 2024-09-14T09:06:37.688 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi164.stderr:2024-09-14T09:06:37.688+0000 7f0bfe298640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x560c3b3aa000 scan_remote_mirror_snapshots: failed to locate remote start snapshot: snap_id=189 2024-09-14T09:06:37.688 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi164.stderr:2024-09-14T09:06:37.688+0000 7f0bfe298640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x560c3b3aa000 scan_remote_mirror_snapshots: split-brain detected: failed to find matching non-primary snapshot in remote image: local_snap_id_start=191, local_snap_ns=[mirror state=non-primary, complete=1, mirror_peer_uuids=, primary_mirror_uuid=9ab74346-7215-4f9b-943c-26e4f257da03, primary_snap_id=bd, last_copied_object_number=128, snap_seqs={189=18446744073709551614}] 2024-09-14T09:06:42.004 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi164.stderr:2024-09-14T09:06:42.003+0000 7f3754609640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55c94be28ea0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-14T09:06:42.004 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi164.stderr:2024-09-14T09:06:42.003+0000 7f375ae16640 -1 rbd::mirror::ImageReplayer: 0x55c94bf16f00 [3/ae9226d8-393a-4b2a-8fd1-1d038e7e5187] operator(): start failed: (85) Interrupted system call should be restarted 2024-09-14T09:06:42.682 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Mirroring disabled 2024-09-14T09:06:42.690 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_present cluster1 mirror image_5 deleted 2024-09-14T09:06:42.691 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:06:42.691 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:06:42.691 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_5 2024-09-14T09:06:42.691 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=deleted 2024-09-14T09:06:42.691 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:06:42.691 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T09:06:42.691 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:06:42.691 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_5 2024-09-14T09:06:42.691 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:06:42.691 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:06:42.691 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_5 2024-09-14T09:06:42.691 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_5 2024-09-14T09:06:42.691 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:06:42.732 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_5: (2) No such file or directory 2024-09-14T09:06:42.736 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image_id= 2024-09-14T09:06:42.736 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T09:06:42.736 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.1 2024-09-14T09:06:42.838 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_present cluster1 mirror image_5 deleted '' 2024-09-14T09:06:42.838 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:06:42.838 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:06:42.838 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_5 2024-09-14T09:06:42.838 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=deleted 2024-09-14T09:06:42.838 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:06:42.838 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=deleted 2024-09-14T09:06:42.838 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_image_id 2024-09-14T09:06:42.838 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_5 2024-09-14T09:06:42.838 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:06:42.838 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:06:42.839 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_5 2024-09-14T09:06:42.839 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_5 2024-09-14T09:06:42.839 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:06:42.875 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_5: (2) No such file or directory 2024-09-14T09:06:42.879 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_image_id= 2024-09-14T09:06:42.879 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:06:42.879 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test deleted = deleted 2024-09-14T09:06:42.879 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:06:42.879 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ purge_snapshots cluster2 mirror image_5 2024-09-14T09:06:42.879 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:06:42.880 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:06:42.880 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_5 2024-09-14T09:06:42.880 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap purge mirror/image_5 2024-09-14T09:06:42.927 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image_retry cluster2 mirror image_5 2024-09-14T09:06:42.927 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:06:42.927 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:06:42.927 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_5 2024-09-14T09:06:42.927 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:06:42.927 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0 2024-09-14T09:06:42.928 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_5 2024-09-14T09:06:42.928 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:06:42.928 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:06:42.928 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_5 2024-09-14T09:06:42.928 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_5 2024-09-14T09:06:42.974 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_5 2024-09-14T09:06:43.019 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 0% complete...failed. 2024-09-14T09:06:43.019 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error: image still has watchers 2024-09-14T09:06:43.019 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout. 2024-09-14T09:06:43.022 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:06:43.022 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T09:06:44.024 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_5 2024-09-14T09:06:44.024 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:06:44.024 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:06:44.024 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_5 2024-09-14T09:06:44.024 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_5 2024-09-14T09:06:44.073 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_5 2024-09-14T09:06:44.120 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 0% complete...failed. 2024-09-14T09:06:44.121 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error: image still has watchers 2024-09-14T09:06:44.121 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout. 2024-09-14T09:06:44.123 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:06:44.124 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 2 2024-09-14T09:06:46.125 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_5 2024-09-14T09:06:46.125 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:06:46.125 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:06:46.125 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_5 2024-09-14T09:06:46.125 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_5 2024-09-14T09:06:46.180 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_5 2024-09-14T09:06:46.229 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 0% complete...failed. 2024-09-14T09:06:46.229 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error: image still has watchers 2024-09-14T09:06:46.229 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout. 2024-09-14T09:06:46.232 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:06:46.232 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 4 2024-09-14T09:06:48.195 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.8072278892661693) > weight (0.25) 2024-09-14T09:06:48.195 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.8560940875401675) > weight (0.25) 2024-09-14T09:06:48.195 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.2 2024-09-14T09:06:48.196 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Sent signal 15 2024-09-14T09:06:48.196 INFO:tasks.rbd_mirror_thrash:waiting for 4 secs before reviving daemons 2024-09-14T09:06:48.197 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi164.stderr:2024-09-14T09:06:48.197+0000 7f2518b36640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 (PID: 62334) UID: 1000 2024-09-14T09:06:50.234 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_5 2024-09-14T09:06:50.234 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:06:50.234 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:06:50.234 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_5 2024-09-14T09:06:50.234 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_5 2024-09-14T09:06:50.288 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_5 2024-09-14T09:06:50.337 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 0% complete...failed. 2024-09-14T09:06:50.337 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error: image still has watchers 2024-09-14T09:06:50.337 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout. 2024-09-14T09:06:50.341 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:06:50.341 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 8 2024-09-14T09:06:52.197 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.2 2024-09-14T09:06:52.197 INFO:teuthology.orchestra.run:waiting for 600 2024-09-14T09:06:52.198 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.2 2024-09-14T09:06:52.198 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Restarting daemon 2024-09-14T09:06:52.198 DEBUG:teuthology.orchestra.run.smithi164:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 2024-09-14T09:06:52.201 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Started 2024-09-14T09:06:52.201 INFO:tasks.rbd_mirror_thrash:waiting for 107 secs before thrashing 2024-09-14T09:06:58.342 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_5 2024-09-14T09:06:58.342 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:06:58.342 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:06:58.342 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_5 2024-09-14T09:06:58.342 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_5 2024-09-14T09:06:58.394 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_5 2024-09-14T09:06:58.442 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 0% complete...failed. 2024-09-14T09:06:58.442 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error: image still has watchers 2024-09-14T09:06:58.442 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout. 2024-09-14T09:06:58.445 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:06:58.445 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 16 2024-09-14T09:07:04.069 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.5787210979474154) > weight (0.25) 2024-09-14T09:07:04.069 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.1 2024-09-14T09:07:04.070 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Sent signal 15 2024-09-14T09:07:04.070 INFO:tasks.rbd_mirror_thrash:waiting for 1 secs before reviving daemons 2024-09-14T09:07:04.071 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi164.stderr:2024-09-14T09:07:04.070+0000 7f7797928640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 (PID: 70225) UID: 1000 2024-09-14T09:07:05.071 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.1 2024-09-14T09:07:05.071 INFO:teuthology.orchestra.run:waiting for 600 2024-09-14T09:07:05.072 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.1 2024-09-14T09:07:05.072 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Restarting daemon 2024-09-14T09:07:05.072 DEBUG:teuthology.orchestra.run.smithi164:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 2024-09-14T09:07:05.075 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Started 2024-09-14T09:07:05.075 INFO:tasks.rbd_mirror_thrash:waiting for 94 secs before thrashing 2024-09-14T09:07:14.447 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_5 2024-09-14T09:07:14.447 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:07:14.447 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:07:14.447 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_5 2024-09-14T09:07:14.447 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_5 2024-09-14T09:07:14.499 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_5 2024-09-14T09:07:14.716 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-14T09:07:14.721 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:07:14.721 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 10` 2024-09-14T09:07:14.721 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_6 2024-09-14T09:07:14.721 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image cluster2 mirror image_6 512M 2024-09-14T09:07:14.721 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:07:14.722 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T09:07:14.722 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:07:14.722 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T09:07:14.722 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_6 2024-09-14T09:07:14.722 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T09:07:14.722 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local size=128 2024-09-14T09:07:14.723 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n 512M ']' 2024-09-14T09:07:14.723 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ size=512M 2024-09-14T09:07:14.723 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T09:07:14.723 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering mirror/image_6 2024-09-14T09:07:14.771 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ enable_mirror cluster2 mirror image_6 2024-09-14T09:07:14.771 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:07:14.771 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:07:14.771 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_6 2024-09-14T09:07:14.772 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T09:07:14.772 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_6 snapshot 2024-09-14T09:07:15.676 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Mirroring enabled 2024-09-14T09:07:15.681 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 info mirror/image_6 2024-09-14T09:07:15.720 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_6': 2024-09-14T09:07:15.721 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 512 MiB in 128 objects 2024-09-14T09:07:15.721 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T09:07:15.721 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 1 2024-09-14T09:07:15.721 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 19ff2f9ff44 2024-09-14T09:07:15.721 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.19ff2f9ff44 2024-09-14T09:07:15.721 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T09:07:15.721 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering 2024-09-14T09:07:15.721 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T09:07:15.721 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T09:07:15.721 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 09:07:14 2024 2024-09-14T09:07:15.721 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 09:07:14 2024 2024-09-14T09:07:15.722 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 09:07:14 2024 2024-09-14T09:07:15.722 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T09:07:15.722 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T09:07:15.722 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: ad12d62b-6b99-42e6-8f1f-553aa7df8494 2024-09-14T09:07:15.722 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: true 2024-09-14T09:07:15.724 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ stress_write_image cluster2 mirror image_6 2024-09-14T09:07:15.724 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:07:15.724 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:07:15.724 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_6 2024-09-14T09:07:15.724 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T09:07:15.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local duration=13 2024-09-14T09:07:15.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set +e 2024-09-14T09:07:15.726 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ timeout 13s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_6 --debug-rbd=20 --debug-journaler=20 2024-09-14T09:07:16.095 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:100 IOs, 101904384 bytes 2024-09-14T09:07:16.250 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:200 IOs, 127372288 bytes 2024-09-14T09:07:16.371 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:300 IOs, 142794752 bytes 2024-09-14T09:07:16.576 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:400 IOs, 211883008 bytes 2024-09-14T09:07:16.804 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:500 IOs, 254128128 bytes 2024-09-14T09:07:16.975 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:600 IOs, 306912256 bytes 2024-09-14T09:07:17.207 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:700 IOs, 365364224 bytes 2024-09-14T09:07:17.296 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:800 IOs, 365989888 bytes 2024-09-14T09:07:17.428 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:900 IOs, 393618432 bytes 2024-09-14T09:07:17.575 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1000 IOs, 418796544 bytes 2024-09-14T09:07:17.733 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1100 IOs, 461458432 bytes 2024-09-14T09:07:17.932 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1200 IOs, 501417984 bytes 2024-09-14T09:07:18.107 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1300 IOs, 542542848 bytes 2024-09-14T09:07:18.288 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1400 IOs, 582545408 bytes 2024-09-14T09:07:18.504 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1500 IOs, 633540608 bytes 2024-09-14T09:07:18.644 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1600 IOs, 658481152 bytes 2024-09-14T09:07:18.801 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1700 IOs, 686245888 bytes 2024-09-14T09:07:19.140 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1800 IOs, 802164736 bytes 2024-09-14T09:07:19.352 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1900 IOs, 854119424 bytes 2024-09-14T09:07:19.446 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2000 IOs, 861622272 bytes 2024-09-14T09:07:19.599 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2100 IOs, 895849472 bytes 2024-09-14T09:07:19.723 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2200 IOs, 915020800 bytes 2024-09-14T09:07:19.916 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2300 IOs, 968685568 bytes 2024-09-14T09:07:20.121 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2400 IOs, 1034765312 bytes 2024-09-14T09:07:20.365 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2500 IOs, 1087991808 bytes 2024-09-14T09:07:20.453 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2600 IOs, 1091468288 bytes 2024-09-14T09:07:20.624 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2700 IOs, 1130921984 bytes 2024-09-14T09:07:20.810 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2800 IOs, 1171314688 bytes 2024-09-14T09:07:21.149 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2900 IOs, 1282847744 bytes 2024-09-14T09:07:21.234 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3000 IOs, 1284895744 bytes 2024-09-14T09:07:21.422 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3100 IOs, 1344648192 bytes 2024-09-14T09:07:21.600 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3200 IOs, 1372216320 bytes 2024-09-14T09:07:21.786 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3300 IOs, 1432542208 bytes 2024-09-14T09:07:22.021 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3400 IOs, 1488479232 bytes 2024-09-14T09:07:22.202 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3500 IOs, 1537086464 bytes 2024-09-14T09:07:22.398 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3600 IOs, 1576296448 bytes 2024-09-14T09:07:22.676 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3700 IOs, 1655417856 bytes 2024-09-14T09:07:22.810 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3800 IOs, 1678240768 bytes 2024-09-14T09:07:22.963 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3900 IOs, 1725869056 bytes 2024-09-14T09:07:23.225 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4000 IOs, 1787236352 bytes 2024-09-14T09:07:23.361 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4100 IOs, 1813686272 bytes 2024-09-14T09:07:23.610 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4200 IOs, 1888535552 bytes 2024-09-14T09:07:23.775 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4300 IOs, 1940382720 bytes 2024-09-14T09:07:24.136 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4400 IOs, 2040519680 bytes 2024-09-14T09:07:24.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4500 IOs, 2098805760 bytes 2024-09-14T09:07:24.525 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4600 IOs, 2134029312 bytes 2024-09-14T09:07:24.712 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4700 IOs, 2180472832 bytes 2024-09-14T09:07:24.894 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4800 IOs, 2221376512 bytes 2024-09-14T09:07:25.066 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4900 IOs, 2261364736 bytes 2024-09-14T09:07:25.194 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5000 IOs, 2284041216 bytes 2024-09-14T09:07:25.284 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5100 IOs, 2284647424 bytes 2024-09-14T09:07:25.413 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5200 IOs, 2306817024 bytes 2024-09-14T09:07:25.678 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5300 IOs, 2383531008 bytes 2024-09-14T09:07:25.832 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5400 IOs, 2428521472 bytes 2024-09-14T09:07:25.989 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5500 IOs, 2462700544 bytes 2024-09-14T09:07:26.217 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5600 IOs, 2510718976 bytes 2024-09-14T09:07:26.306 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5700 IOs, 2524797952 bytes 2024-09-14T09:07:26.409 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5800 IOs, 2525804544 bytes 2024-09-14T09:07:26.559 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5900 IOs, 2559997952 bytes 2024-09-14T09:07:26.790 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6000 IOs, 2616139776 bytes 2024-09-14T09:07:26.912 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6100 IOs, 2649976832 bytes 2024-09-14T09:07:27.163 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6200 IOs, 2722314240 bytes 2024-09-14T09:07:27.558 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6300 IOs, 2836221952 bytes 2024-09-14T09:07:27.787 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6400 IOs, 2895011840 bytes 2024-09-14T09:07:27.912 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6500 IOs, 2912770048 bytes 2024-09-14T09:07:28.112 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6600 IOs, 2958979072 bytes 2024-09-14T09:07:28.308 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6700 IOs, 3008977920 bytes 2024-09-14T09:07:28.486 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6800 IOs, 3064992768 bytes 2024-09-14T09:07:28.606 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6900 IOs, 3065609216 bytes 2024-09-14T09:07:28.736 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ error_code=124 2024-09-14T09:07:28.736 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set -e 2024-09-14T09:07:28.736 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T09:07:28.736 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:07:28.736 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_present cluster1 mirror image_6 present 2024-09-14T09:07:28.736 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:07:28.736 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:07:28.736 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_6 2024-09-14T09:07:28.737 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=present 2024-09-14T09:07:28.737 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:07:28.737 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T09:07:28.737 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:07:28.737 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_6 2024-09-14T09:07:28.737 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:07:28.737 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:07:28.737 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_6 2024-09-14T09:07:28.737 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_6 2024-09-14T09:07:28.737 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:07:28.788 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image_id=15393778fa 2024-09-14T09:07:28.789 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T09:07:28.789 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.1 2024-09-14T09:07:28.889 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_present cluster1 mirror image_6 present 15393778fa 2024-09-14T09:07:28.890 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:07:28.890 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:07:28.890 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_6 2024-09-14T09:07:28.890 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=present 2024-09-14T09:07:28.890 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id=15393778fa 2024-09-14T09:07:28.890 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=deleted 2024-09-14T09:07:28.890 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_image_id 2024-09-14T09:07:28.890 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_6 2024-09-14T09:07:28.890 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:07:28.890 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:07:28.890 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_6 2024-09-14T09:07:28.890 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_6 2024-09-14T09:07:28.890 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:07:28.935 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_image_id=15393778fa 2024-09-14T09:07:28.935 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n 15393778fa 2024-09-14T09:07:28.935 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -z 15393778fa -o 15393778fa = 15393778fa 2024-09-14T09:07:28.935 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_state=present 2024-09-14T09:07:28.935 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test present = present 2024-09-14T09:07:28.936 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:07:28.936 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ disable_mirror cluster2 mirror image_6 2024-09-14T09:07:28.936 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:07:28.936 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:07:28.936 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_6 2024-09-14T09:07:28.936 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_6 2024-09-14T09:07:29.686 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi164.stderr:2024-09-14T09:07:29.686+0000 7f0bfea99640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x560c3a7c2000 scan_remote_mirror_snapshots: failed to locate remote start snapshot: snap_id=190 2024-09-14T09:07:29.687 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi164.stderr:2024-09-14T09:07:29.686+0000 7f0bfea99640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x560c3a7c2000 scan_remote_mirror_snapshots: split-brain detected: failed to find matching non-primary snapshot in remote image: local_snap_id_start=192, local_snap_ns=[mirror state=non-primary, complete=1, mirror_peer_uuids=, primary_mirror_uuid=9ab74346-7215-4f9b-943c-26e4f257da03, primary_snap_id=be, last_copied_object_number=128, snap_seqs={190=18446744073709551614}] 2024-09-14T09:07:34.687 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Mirroring disabled 2024-09-14T09:07:34.697 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_present cluster1 mirror image_6 deleted 2024-09-14T09:07:34.697 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:07:34.697 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:07:34.697 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_6 2024-09-14T09:07:34.697 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=deleted 2024-09-14T09:07:34.697 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:07:34.698 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T09:07:34.698 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:07:34.698 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_6 2024-09-14T09:07:34.698 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:07:34.698 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:07:34.698 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_6 2024-09-14T09:07:34.698 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_6 2024-09-14T09:07:34.698 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:07:34.739 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_6: (2) No such file or directory 2024-09-14T09:07:34.743 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image_id= 2024-09-14T09:07:34.743 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T09:07:34.743 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.1 2024-09-14T09:07:34.845 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_present cluster1 mirror image_6 deleted '' 2024-09-14T09:07:34.845 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:07:34.845 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:07:34.845 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_6 2024-09-14T09:07:34.845 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=deleted 2024-09-14T09:07:34.845 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:07:34.845 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=deleted 2024-09-14T09:07:34.845 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_image_id 2024-09-14T09:07:34.845 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_6 2024-09-14T09:07:34.845 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:07:34.846 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:07:34.846 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_6 2024-09-14T09:07:34.846 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_6 2024-09-14T09:07:34.846 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:07:34.884 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_6: (2) No such file or directory 2024-09-14T09:07:34.888 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_image_id= 2024-09-14T09:07:34.888 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:07:34.888 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test deleted = deleted 2024-09-14T09:07:34.888 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:07:34.888 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ purge_snapshots cluster2 mirror image_6 2024-09-14T09:07:34.888 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:07:34.888 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:07:34.888 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_6 2024-09-14T09:07:34.888 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap purge mirror/image_6 2024-09-14T09:07:34.936 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image_retry cluster2 mirror image_6 2024-09-14T09:07:34.936 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:07:34.936 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:07:34.936 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_6 2024-09-14T09:07:34.936 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:07:34.936 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0 2024-09-14T09:07:34.937 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_6 2024-09-14T09:07:34.937 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:07:34.937 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:07:34.937 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_6 2024-09-14T09:07:34.937 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_6 2024-09-14T09:07:34.983 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_6 2024-09-14T09:07:35.028 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 0% complete...failed. 2024-09-14T09:07:35.029 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error: image still has watchers 2024-09-14T09:07:35.029 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout. 2024-09-14T09:07:35.032 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:07:35.032 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T09:07:36.033 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_6 2024-09-14T09:07:36.033 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:07:36.033 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:07:36.033 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_6 2024-09-14T09:07:36.034 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_6 2024-09-14T09:07:36.089 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_6 2024-09-14T09:07:36.138 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 0% complete...failed. 2024-09-14T09:07:36.138 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error: image still has watchers 2024-09-14T09:07:36.138 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout. 2024-09-14T09:07:36.141 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:07:36.141 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 2 2024-09-14T09:07:38.143 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_6 2024-09-14T09:07:38.143 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:07:38.143 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:07:38.143 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_6 2024-09-14T09:07:38.143 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_6 2024-09-14T09:07:38.199 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_6 2024-09-14T09:07:38.248 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 0% complete...failed. 2024-09-14T09:07:38.248 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error: image still has watchers 2024-09-14T09:07:38.248 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout. 2024-09-14T09:07:38.251 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:07:38.251 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 4 2024-09-14T09:07:42.253 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_6 2024-09-14T09:07:42.253 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:07:42.253 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:07:42.253 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_6 2024-09-14T09:07:42.253 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_6 2024-09-14T09:07:42.307 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_6 2024-09-14T09:07:42.357 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 0% complete...failed. 2024-09-14T09:07:42.357 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error: image still has watchers 2024-09-14T09:07:42.357 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout. 2024-09-14T09:07:42.360 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:07:42.360 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 8 2024-09-14T09:07:50.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_6 2024-09-14T09:07:50.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:07:50.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:07:50.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_6 2024-09-14T09:07:50.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_6 2024-09-14T09:07:50.415 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_6 2024-09-14T09:07:50.464 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 0% complete...failed. 2024-09-14T09:07:50.465 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error: image still has watchers 2024-09-14T09:07:50.465 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout. 2024-09-14T09:07:50.467 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:07:50.467 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 16 2024-09-14T09:08:06.469 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_6 2024-09-14T09:08:06.469 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:08:06.469 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:08:06.469 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_6 2024-09-14T09:08:06.470 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_6 2024-09-14T09:08:06.525 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_6 2024-09-14T09:08:06.932 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-14T09:08:06.935 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:08:06.936 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 10` 2024-09-14T09:08:06.936 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_7 2024-09-14T09:08:06.936 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image cluster2 mirror image_7 512M 2024-09-14T09:08:06.936 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:08:06.936 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T09:08:06.936 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:08:06.936 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T09:08:06.936 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_7 2024-09-14T09:08:06.936 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T09:08:06.936 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local size=128 2024-09-14T09:08:06.936 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n 512M ']' 2024-09-14T09:08:06.936 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ size=512M 2024-09-14T09:08:06.937 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T09:08:06.937 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering mirror/image_7 2024-09-14T09:08:06.985 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ enable_mirror cluster2 mirror image_7 2024-09-14T09:08:06.985 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:08:06.985 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:08:06.985 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_7 2024-09-14T09:08:06.985 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T09:08:06.985 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_7 snapshot 2024-09-14T09:08:07.728 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Mirroring enabled 2024-09-14T09:08:07.735 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 info mirror/image_7 2024-09-14T09:08:07.776 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_7': 2024-09-14T09:08:07.776 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 512 MiB in 128 objects 2024-09-14T09:08:07.776 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T09:08:07.776 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 1 2024-09-14T09:08:07.776 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 1a35d1813558 2024-09-14T09:08:07.777 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.1a35d1813558 2024-09-14T09:08:07.777 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T09:08:07.777 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering 2024-09-14T09:08:07.777 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T09:08:07.777 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T09:08:07.777 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 09:08:06 2024 2024-09-14T09:08:07.777 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 09:08:06 2024 2024-09-14T09:08:07.777 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 09:08:06 2024 2024-09-14T09:08:07.777 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T09:08:07.777 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T09:08:07.777 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: 8817070b-dfa6-47d0-b6bc-d2f2e037eb99 2024-09-14T09:08:07.777 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: true 2024-09-14T09:08:07.780 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ stress_write_image cluster2 mirror image_7 2024-09-14T09:08:07.780 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:08:07.780 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:08:07.780 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_7 2024-09-14T09:08:07.780 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T09:08:07.782 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local duration=8 2024-09-14T09:08:07.782 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set +e 2024-09-14T09:08:07.782 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ timeout 8s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_7 --debug-rbd=20 --debug-journaler=20 2024-09-14T09:08:08.156 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:100 IOs, 109309952 bytes 2024-09-14T09:08:08.259 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:200 IOs, 119949312 bytes 2024-09-14T09:08:08.362 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:300 IOs, 139366400 bytes 2024-09-14T09:08:08.575 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:400 IOs, 188756992 bytes 2024-09-14T09:08:08.705 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:500 IOs, 210068480 bytes 2024-09-14T09:08:08.887 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:600 IOs, 255971328 bytes 2024-09-14T09:08:09.022 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:700 IOs, 278864896 bytes 2024-09-14T09:08:09.198 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:800 IOs, 324812800 bytes 2024-09-14T09:08:09.372 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:900 IOs, 370089984 bytes 2024-09-14T09:08:09.556 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1000 IOs, 402432000 bytes 2024-09-14T09:08:09.873 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1100 IOs, 504613888 bytes 2024-09-14T09:08:10.074 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1200 IOs, 550272000 bytes 2024-09-14T09:08:10.222 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1300 IOs, 577474560 bytes 2024-09-14T09:08:10.390 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1400 IOs, 617349120 bytes 2024-09-14T09:08:10.518 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1500 IOs, 635536384 bytes 2024-09-14T09:08:10.663 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1600 IOs, 660734976 bytes 2024-09-14T09:08:10.743 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1700 IOs, 661338112 bytes 2024-09-14T09:08:10.883 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1800 IOs, 691532800 bytes 2024-09-14T09:08:11.175 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1900 IOs, 789712896 bytes 2024-09-14T09:08:11.336 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2000 IOs, 809696256 bytes 2024-09-14T09:08:11.498 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2100 IOs, 845847552 bytes 2024-09-14T09:08:11.718 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2200 IOs, 907272192 bytes 2024-09-14T09:08:11.912 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2300 IOs, 957715456 bytes 2024-09-14T09:08:12.103 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2400 IOs, 1002942464 bytes 2024-09-14T09:08:12.339 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2500 IOs, 1069175808 bytes 2024-09-14T09:08:12.530 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2600 IOs, 1115648000 bytes 2024-09-14T09:08:12.730 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2700 IOs, 1164426240 bytes 2024-09-14T09:08:12.885 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2800 IOs, 1197632512 bytes 2024-09-14T09:08:13.039 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2900 IOs, 1231864832 bytes 2024-09-14T09:08:13.122 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3000 IOs, 1232468992 bytes 2024-09-14T09:08:13.208 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3100 IOs, 1235018752 bytes 2024-09-14T09:08:13.426 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3200 IOs, 1314443264 bytes 2024-09-14T09:08:13.588 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3300 IOs, 1331122176 bytes 2024-09-14T09:08:13.692 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3400 IOs, 1351675904 bytes 2024-09-14T09:08:13.924 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3500 IOs, 1396165632 bytes 2024-09-14T09:08:14.022 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3600 IOs, 1404396544 bytes 2024-09-14T09:08:14.108 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3700 IOs, 1407733760 bytes 2024-09-14T09:08:14.242 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3800 IOs, 1430050816 bytes 2024-09-14T09:08:14.387 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3900 IOs, 1454687232 bytes 2024-09-14T09:08:14.476 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4000 IOs, 1463629824 bytes 2024-09-14T09:08:14.605 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4100 IOs, 1482655744 bytes 2024-09-14T09:08:14.805 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4200 IOs, 1537382400 bytes 2024-09-14T09:08:14.921 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4300 IOs, 1553363968 bytes 2024-09-14T09:08:15.200 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4400 IOs, 1635009536 bytes 2024-09-14T09:08:15.369 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4500 IOs, 1669230592 bytes 2024-09-14T09:08:15.551 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4600 IOs, 1740054528 bytes 2024-09-14T09:08:15.719 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4700 IOs, 1747523584 bytes 2024-09-14T09:08:15.787 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ error_code=124 2024-09-14T09:08:15.788 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set -e 2024-09-14T09:08:15.788 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T09:08:15.788 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:08:15.788 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_present cluster1 mirror image_7 present 2024-09-14T09:08:15.788 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:08:15.788 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:08:15.788 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_7 2024-09-14T09:08:15.788 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=present 2024-09-14T09:08:15.788 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:08:15.788 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T09:08:15.788 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:08:15.788 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_7 2024-09-14T09:08:15.788 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:08:15.788 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:08:15.789 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_7 2024-09-14T09:08:15.789 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_7 2024-09-14T09:08:15.789 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:08:15.844 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image_id=153916ad0527 2024-09-14T09:08:15.844 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T09:08:15.844 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.1 2024-09-14T09:08:15.945 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_present cluster1 mirror image_7 present 153916ad0527 2024-09-14T09:08:15.946 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:08:15.946 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:08:15.946 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_7 2024-09-14T09:08:15.946 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=present 2024-09-14T09:08:15.946 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id=153916ad0527 2024-09-14T09:08:15.946 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=deleted 2024-09-14T09:08:15.946 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_image_id 2024-09-14T09:08:15.946 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_7 2024-09-14T09:08:15.946 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:08:15.946 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:08:15.946 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_7 2024-09-14T09:08:15.946 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_7 2024-09-14T09:08:15.947 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:08:15.991 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_image_id=153916ad0527 2024-09-14T09:08:15.991 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n 153916ad0527 2024-09-14T09:08:15.991 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -z 153916ad0527 -o 153916ad0527 = 153916ad0527 2024-09-14T09:08:15.991 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_state=present 2024-09-14T09:08:15.991 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test present = present 2024-09-14T09:08:15.991 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:08:15.991 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ disable_mirror cluster2 mirror image_7 2024-09-14T09:08:15.992 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:08:15.992 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:08:15.992 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_7 2024-09-14T09:08:15.992 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_7 2024-09-14T09:08:16.737 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi164.stderr:2024-09-14T09:08:16.737+0000 7f0bfea99640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x560c3b187c00 scan_remote_mirror_snapshots: failed to locate remote start snapshot: snap_id=191 2024-09-14T09:08:16.737 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi164.stderr:2024-09-14T09:08:16.737+0000 7f0bfea99640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x560c3b187c00 scan_remote_mirror_snapshots: split-brain detected: failed to find matching non-primary snapshot in remote image: local_snap_id_start=193, local_snap_ns=[mirror state=non-primary, complete=1, mirror_peer_uuids=, primary_mirror_uuid=9ab74346-7215-4f9b-943c-26e4f257da03, primary_snap_id=bf, last_copied_object_number=128, snap_seqs={191=18446744073709551614}] 2024-09-14T09:08:21.739 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Mirroring disabled 2024-09-14T09:08:21.747 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_present cluster1 mirror image_7 deleted 2024-09-14T09:08:21.747 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:08:21.747 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:08:21.747 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_7 2024-09-14T09:08:21.747 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=deleted 2024-09-14T09:08:21.747 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:08:21.748 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T09:08:21.748 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:08:21.748 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_7 2024-09-14T09:08:21.748 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:08:21.748 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:08:21.748 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_7 2024-09-14T09:08:21.748 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_7 2024-09-14T09:08:21.748 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:08:21.789 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_7: (2) No such file or directory 2024-09-14T09:08:21.792 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image_id= 2024-09-14T09:08:21.792 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T09:08:21.792 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.1 2024-09-14T09:08:21.894 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_present cluster1 mirror image_7 deleted '' 2024-09-14T09:08:21.894 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:08:21.894 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:08:21.894 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_7 2024-09-14T09:08:21.895 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=deleted 2024-09-14T09:08:21.895 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:08:21.895 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=deleted 2024-09-14T09:08:21.895 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_image_id 2024-09-14T09:08:21.895 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_7 2024-09-14T09:08:21.895 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:08:21.895 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:08:21.895 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_7 2024-09-14T09:08:21.895 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_7 2024-09-14T09:08:21.895 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:08:21.932 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_7: (2) No such file or directory 2024-09-14T09:08:21.935 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_image_id= 2024-09-14T09:08:21.935 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:08:21.935 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test deleted = deleted 2024-09-14T09:08:21.935 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:08:21.936 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ purge_snapshots cluster2 mirror image_7 2024-09-14T09:08:21.936 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:08:21.936 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:08:21.936 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_7 2024-09-14T09:08:21.936 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap purge mirror/image_7 2024-09-14T09:08:21.982 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image_retry cluster2 mirror image_7 2024-09-14T09:08:21.982 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:08:21.982 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:08:21.982 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_7 2024-09-14T09:08:21.982 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:08:21.982 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0 2024-09-14T09:08:21.983 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_7 2024-09-14T09:08:21.983 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:08:21.983 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:08:21.983 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_7 2024-09-14T09:08:21.983 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_7 2024-09-14T09:08:22.028 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_7 2024-09-14T09:08:22.073 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 0% complete...failed. 2024-09-14T09:08:22.073 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error: image still has watchers 2024-09-14T09:08:22.073 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout. 2024-09-14T09:08:22.076 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:08:22.076 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T09:08:23.077 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_7 2024-09-14T09:08:23.078 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:08:23.078 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:08:23.078 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_7 2024-09-14T09:08:23.078 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_7 2024-09-14T09:08:23.127 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_7 2024-09-14T09:08:23.173 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 0% complete...failed. 2024-09-14T09:08:23.173 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error: image still has watchers 2024-09-14T09:08:23.173 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout. 2024-09-14T09:08:23.176 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:08:23.176 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 2 2024-09-14T09:08:25.177 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_7 2024-09-14T09:08:25.178 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:08:25.178 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:08:25.178 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_7 2024-09-14T09:08:25.178 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_7 2024-09-14T09:08:25.233 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_7 2024-09-14T09:08:25.281 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 0% complete...failed. 2024-09-14T09:08:25.281 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error: image still has watchers 2024-09-14T09:08:25.281 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout. 2024-09-14T09:08:25.284 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:08:25.284 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 4 2024-09-14T09:08:29.286 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_7 2024-09-14T09:08:29.286 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:08:29.286 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:08:29.286 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_7 2024-09-14T09:08:29.287 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_7 2024-09-14T09:08:29.339 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_7 2024-09-14T09:08:29.387 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 0% complete...failed. 2024-09-14T09:08:29.387 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error: image still has watchers 2024-09-14T09:08:29.387 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout. 2024-09-14T09:08:29.390 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:08:29.390 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 8 2024-09-14T09:08:37.392 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_7 2024-09-14T09:08:37.392 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:08:37.392 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:08:37.392 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_7 2024-09-14T09:08:37.392 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_7 2024-09-14T09:08:37.447 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_7 2024-09-14T09:08:37.495 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 0% complete...failed. 2024-09-14T09:08:37.495 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error: image still has watchers 2024-09-14T09:08:37.495 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout. 2024-09-14T09:08:37.498 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:08:37.498 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 16 2024-09-14T09:08:39.076 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.493167004093529) > weight (0.25) 2024-09-14T09:08:39.076 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.4315215681094938) > weight (0.25) 2024-09-14T09:08:39.076 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.6048870754301567) > weight (0.25) 2024-09-14T09:08:39.076 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.3 with skip (0.5395290848821012) > weight (0.25) 2024-09-14T09:08:39.076 INFO:tasks.rbd_mirror_thrash:waiting for 86 secs before thrashing 2024-09-14T09:08:39.203 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.739289266837641) > weight (0.25) 2024-09-14T09:08:39.203 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.8015126628529613) > weight (0.25) 2024-09-14T09:08:39.203 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.2 2024-09-14T09:08:39.203 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Sent signal 15 2024-09-14T09:08:39.204 INFO:tasks.rbd_mirror_thrash:waiting for 5 secs before reviving daemons 2024-09-14T09:08:39.204 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi164.stderr:2024-09-14T09:08:39.204+0000 7f6ca5d79640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 (PID: 71017) UID: 1000 2024-09-14T09:08:44.205 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.2 2024-09-14T09:08:44.205 INFO:teuthology.orchestra.run:waiting for 600 2024-09-14T09:08:44.205 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.2 2024-09-14T09:08:44.205 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Restarting daemon 2024-09-14T09:08:44.206 DEBUG:teuthology.orchestra.run.smithi164:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 2024-09-14T09:08:44.209 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Started 2024-09-14T09:08:44.209 INFO:tasks.rbd_mirror_thrash:waiting for 119 secs before thrashing 2024-09-14T09:08:53.499 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_7 2024-09-14T09:08:53.500 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:08:53.500 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:08:53.500 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_7 2024-09-14T09:08:53.500 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_7 2024-09-14T09:08:53.551 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_7 2024-09-14T09:08:53.765 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-14T09:08:53.769 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:08:53.769 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 10` 2024-09-14T09:08:53.769 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_8 2024-09-14T09:08:53.769 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image cluster2 mirror image_8 512M 2024-09-14T09:08:53.769 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:08:53.769 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T09:08:53.769 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:08:53.769 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T09:08:53.769 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_8 2024-09-14T09:08:53.769 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T09:08:53.769 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local size=128 2024-09-14T09:08:53.770 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n 512M ']' 2024-09-14T09:08:53.770 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ size=512M 2024-09-14T09:08:53.770 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T09:08:53.770 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering mirror/image_8 2024-09-14T09:08:53.818 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ enable_mirror cluster2 mirror image_8 2024-09-14T09:08:53.818 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:08:53.818 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:08:53.818 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_8 2024-09-14T09:08:53.818 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T09:08:53.818 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_8 snapshot 2024-09-14T09:08:54.691 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Mirroring enabled 2024-09-14T09:08:54.699 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 info mirror/image_8 2024-09-14T09:08:54.740 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_8': 2024-09-14T09:08:54.740 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 512 MiB in 128 objects 2024-09-14T09:08:54.740 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T09:08:54.740 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 1 2024-09-14T09:08:54.740 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 1a705bfd98ee 2024-09-14T09:08:54.740 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.1a705bfd98ee 2024-09-14T09:08:54.740 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T09:08:54.741 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering 2024-09-14T09:08:54.741 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T09:08:54.741 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T09:08:54.741 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 09:08:53 2024 2024-09-14T09:08:54.741 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 09:08:53 2024 2024-09-14T09:08:54.741 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 09:08:53 2024 2024-09-14T09:08:54.741 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T09:08:54.741 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T09:08:54.741 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: f6502c41-7942-42c4-a35b-db47f9bf6020 2024-09-14T09:08:54.741 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: true 2024-09-14T09:08:54.743 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ stress_write_image cluster2 mirror image_8 2024-09-14T09:08:54.743 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:08:54.743 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:08:54.743 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_8 2024-09-14T09:08:54.743 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T09:08:54.745 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local duration=6 2024-09-14T09:08:54.745 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set +e 2024-09-14T09:08:54.745 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ timeout 6s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_8 --debug-rbd=20 --debug-journaler=20 2024-09-14T09:08:55.065 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:100 IOs, 82659328 bytes 2024-09-14T09:08:55.204 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:200 IOs, 107616256 bytes 2024-09-14T09:08:55.325 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:300 IOs, 125937664 bytes 2024-09-14T09:08:55.603 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:400 IOs, 208054272 bytes 2024-09-14T09:08:55.765 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:500 IOs, 240781312 bytes 2024-09-14T09:08:55.905 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:600 IOs, 279672832 bytes 2024-09-14T09:08:56.119 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:700 IOs, 334017536 bytes 2024-09-14T09:08:56.350 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:800 IOs, 386626560 bytes 2024-09-14T09:08:56.471 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:900 IOs, 420958208 bytes 2024-09-14T09:08:56.674 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1000 IOs, 457999360 bytes 2024-09-14T09:08:56.830 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1100 IOs, 488494080 bytes 2024-09-14T09:08:56.998 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1200 IOs, 525875200 bytes 2024-09-14T09:08:57.108 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1300 IOs, 538396672 bytes 2024-09-14T09:08:57.226 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1400 IOs, 556498944 bytes 2024-09-14T09:08:57.420 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1500 IOs, 605847552 bytes 2024-09-14T09:08:57.522 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1600 IOs, 611436544 bytes 2024-09-14T09:08:57.839 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1700 IOs, 712154112 bytes 2024-09-14T09:08:57.963 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1800 IOs, 732773376 bytes 2024-09-14T09:08:58.127 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1900 IOs, 758254592 bytes 2024-09-14T09:08:58.239 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2000 IOs, 779303936 bytes 2024-09-14T09:08:58.399 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2100 IOs, 815294464 bytes 2024-09-14T09:08:58.677 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2200 IOs, 920552448 bytes 2024-09-14T09:08:58.901 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2300 IOs, 951438336 bytes 2024-09-14T09:08:59.146 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2400 IOs, 1020956672 bytes 2024-09-14T09:08:59.302 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2500 IOs, 1059002368 bytes 2024-09-14T09:08:59.523 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2600 IOs, 1115079680 bytes 2024-09-14T09:08:59.914 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2700 IOs, 1247942656 bytes 2024-09-14T09:09:00.119 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2800 IOs, 1291407360 bytes 2024-09-14T09:09:00.359 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2900 IOs, 1362096128 bytes 2024-09-14T09:09:00.474 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3000 IOs, 1389295616 bytes 2024-09-14T09:09:00.753 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ error_code=124 2024-09-14T09:09:00.753 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set -e 2024-09-14T09:09:00.753 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T09:09:00.753 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:09:00.753 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_present cluster1 mirror image_8 present 2024-09-14T09:09:00.753 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:09:00.753 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:09:00.754 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_8 2024-09-14T09:09:00.754 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=present 2024-09-14T09:09:00.754 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:09:00.754 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T09:09:00.754 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:09:00.754 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_8 2024-09-14T09:09:00.754 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:09:00.754 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:09:00.754 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_8 2024-09-14T09:09:00.754 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_8 2024-09-14T09:09:00.754 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:09:00.804 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image_id=1539da047893 2024-09-14T09:09:00.804 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T09:09:00.804 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.1 2024-09-14T09:09:00.906 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_present cluster1 mirror image_8 present 1539da047893 2024-09-14T09:09:00.906 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:09:00.906 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:09:00.906 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_8 2024-09-14T09:09:00.906 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=present 2024-09-14T09:09:00.906 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id=1539da047893 2024-09-14T09:09:00.906 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=deleted 2024-09-14T09:09:00.906 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_image_id 2024-09-14T09:09:00.907 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_8 2024-09-14T09:09:00.907 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:09:00.907 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:09:00.907 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_8 2024-09-14T09:09:00.907 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_8 2024-09-14T09:09:00.907 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:09:01.155 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_image_id=1539da047893 2024-09-14T09:09:01.156 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n 1539da047893 2024-09-14T09:09:01.156 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -z 1539da047893 -o 1539da047893 = 1539da047893 2024-09-14T09:09:01.156 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_state=present 2024-09-14T09:09:01.156 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test present = present 2024-09-14T09:09:01.156 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:09:01.156 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ disable_mirror cluster2 mirror image_8 2024-09-14T09:09:01.156 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:09:01.156 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:09:01.156 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_8 2024-09-14T09:09:01.156 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_8 2024-09-14T09:09:01.693 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi164.stderr:2024-09-14T09:09:01.693+0000 7f0bfea99640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x560c3b99e800 scan_remote_mirror_snapshots: failed to locate remote start snapshot: snap_id=192 2024-09-14T09:09:01.693 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi164.stderr:2024-09-14T09:09:01.693+0000 7f0bfea99640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x560c3b99e800 scan_remote_mirror_snapshots: split-brain detected: failed to find matching non-primary snapshot in remote image: local_snap_id_start=194, local_snap_ns=[mirror state=non-primary, complete=1, mirror_peer_uuids=, primary_mirror_uuid=9ab74346-7215-4f9b-943c-26e4f257da03, primary_snap_id=c0, last_copied_object_number=128, snap_seqs={192=18446744073709551614}] 2024-09-14T09:09:06.696 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Mirroring disabled 2024-09-14T09:09:06.704 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_present cluster1 mirror image_8 deleted 2024-09-14T09:09:06.704 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:09:06.705 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:09:06.705 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_8 2024-09-14T09:09:06.705 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=deleted 2024-09-14T09:09:06.705 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:09:06.705 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T09:09:06.705 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:09:06.705 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_8 2024-09-14T09:09:06.705 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:09:06.705 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:09:06.705 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_8 2024-09-14T09:09:06.706 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_8 2024-09-14T09:09:06.706 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:09:06.747 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_8: (2) No such file or directory 2024-09-14T09:09:06.751 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image_id= 2024-09-14T09:09:06.751 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T09:09:06.751 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.1 2024-09-14T09:09:06.852 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_present cluster1 mirror image_8 deleted '' 2024-09-14T09:09:06.853 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:09:06.853 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:09:06.853 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_8 2024-09-14T09:09:06.853 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=deleted 2024-09-14T09:09:06.853 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:09:06.853 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=deleted 2024-09-14T09:09:06.853 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_image_id 2024-09-14T09:09:06.853 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_8 2024-09-14T09:09:06.853 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:09:06.853 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:09:06.853 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_8 2024-09-14T09:09:06.853 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_8 2024-09-14T09:09:06.853 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:09:06.890 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_8: (2) No such file or directory 2024-09-14T09:09:06.894 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_image_id= 2024-09-14T09:09:06.894 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:09:06.894 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test deleted = deleted 2024-09-14T09:09:06.894 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:09:06.894 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ purge_snapshots cluster2 mirror image_8 2024-09-14T09:09:06.894 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:09:06.894 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:09:06.894 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_8 2024-09-14T09:09:06.895 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap purge mirror/image_8 2024-09-14T09:09:06.941 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image_retry cluster2 mirror image_8 2024-09-14T09:09:06.941 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:09:06.941 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:09:06.941 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_8 2024-09-14T09:09:06.941 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:09:06.941 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0 2024-09-14T09:09:06.942 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_8 2024-09-14T09:09:06.942 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:09:06.942 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:09:06.942 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_8 2024-09-14T09:09:06.942 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_8 2024-09-14T09:09:06.987 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_8 2024-09-14T09:09:07.033 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 0% complete...failed. 2024-09-14T09:09:07.033 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error: image still has watchers 2024-09-14T09:09:07.033 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout. 2024-09-14T09:09:07.036 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:09:07.036 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T09:09:08.038 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_8 2024-09-14T09:09:08.038 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:09:08.038 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:09:08.038 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_8 2024-09-14T09:09:08.038 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_8 2024-09-14T09:09:08.089 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_8 2024-09-14T09:09:08.135 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 0% complete...failed. 2024-09-14T09:09:08.136 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error: image still has watchers 2024-09-14T09:09:08.136 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout. 2024-09-14T09:09:08.139 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:09:08.139 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 2 2024-09-14T09:09:10.140 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_8 2024-09-14T09:09:10.140 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:09:10.140 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:09:10.141 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_8 2024-09-14T09:09:10.141 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_8 2024-09-14T09:09:10.193 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_8 2024-09-14T09:09:10.243 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 0% complete...failed. 2024-09-14T09:09:10.243 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error: image still has watchers 2024-09-14T09:09:10.243 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout. 2024-09-14T09:09:10.247 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:09:10.247 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 4 2024-09-14T09:09:14.248 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_8 2024-09-14T09:09:14.248 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:09:14.248 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:09:14.249 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_8 2024-09-14T09:09:14.249 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_8 2024-09-14T09:09:14.299 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_8 2024-09-14T09:09:14.347 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 0% complete...failed. 2024-09-14T09:09:14.347 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error: image still has watchers 2024-09-14T09:09:14.347 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout. 2024-09-14T09:09:14.350 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:09:14.350 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 8 2024-09-14T09:09:22.352 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_8 2024-09-14T09:09:22.352 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:09:22.352 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:09:22.352 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_8 2024-09-14T09:09:22.352 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_8 2024-09-14T09:09:22.404 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_8 2024-09-14T09:09:22.450 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 0% complete...failed. 2024-09-14T09:09:22.450 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error: image still has watchers 2024-09-14T09:09:22.450 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout. 2024-09-14T09:09:22.453 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:09:22.453 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 16 2024-09-14T09:09:38.455 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_8 2024-09-14T09:09:38.455 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:09:38.455 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:09:38.455 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_8 2024-09-14T09:09:38.456 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_8 2024-09-14T09:09:38.510 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_8 2024-09-14T09:09:38.725 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-14T09:09:38.729 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:09:38.729 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 10` 2024-09-14T09:09:38.729 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_9 2024-09-14T09:09:38.729 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image cluster2 mirror image_9 512M 2024-09-14T09:09:38.730 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:09:38.730 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T09:09:38.730 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:09:38.730 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T09:09:38.730 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_9 2024-09-14T09:09:38.730 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T09:09:38.730 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local size=128 2024-09-14T09:09:38.730 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n 512M ']' 2024-09-14T09:09:38.730 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ size=512M 2024-09-14T09:09:38.730 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T09:09:38.730 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering mirror/image_9 2024-09-14T09:09:38.779 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ enable_mirror cluster2 mirror image_9 2024-09-14T09:09:38.779 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:09:38.779 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:09:38.779 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_9 2024-09-14T09:09:38.779 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T09:09:38.780 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_9 snapshot 2024-09-14T09:09:39.746 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Mirroring enabled 2024-09-14T09:09:39.751 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 info mirror/image_9 2024-09-14T09:09:39.790 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_9': 2024-09-14T09:09:39.790 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 512 MiB in 128 objects 2024-09-14T09:09:39.790 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T09:09:39.790 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 1 2024-09-14T09:09:39.790 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 1aa69df017d6 2024-09-14T09:09:39.790 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.1aa69df017d6 2024-09-14T09:09:39.790 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T09:09:39.790 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering 2024-09-14T09:09:39.790 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T09:09:39.791 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T09:09:39.791 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 09:09:38 2024 2024-09-14T09:09:39.791 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 09:09:38 2024 2024-09-14T09:09:39.791 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 09:09:38 2024 2024-09-14T09:09:39.791 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T09:09:39.791 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T09:09:39.791 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: 8cc15ce4-f134-4e45-aae8-b3e05b1aafb1 2024-09-14T09:09:39.791 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: true 2024-09-14T09:09:39.793 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ stress_write_image cluster2 mirror image_9 2024-09-14T09:09:39.793 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:09:39.793 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:09:39.793 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_9 2024-09-14T09:09:39.793 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T09:09:39.795 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local duration=5 2024-09-14T09:09:39.795 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set +e 2024-09-14T09:09:39.795 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ timeout 5s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_9 --debug-rbd=20 --debug-journaler=20 2024-09-14T09:09:40.039 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:100 IOs, 62892032 bytes 2024-09-14T09:09:40.299 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:200 IOs, 134996992 bytes 2024-09-14T09:09:40.486 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:300 IOs, 175784960 bytes 2024-09-14T09:09:40.669 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:400 IOs, 222328832 bytes 2024-09-14T09:09:40.851 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:500 IOs, 261401600 bytes 2024-09-14T09:09:41.125 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:600 IOs, 348114944 bytes 2024-09-14T09:09:41.214 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:700 IOs, 355493888 bytes 2024-09-14T09:09:41.431 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:800 IOs, 422562816 bytes 2024-09-14T09:09:41.642 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:900 IOs, 465397760 bytes 2024-09-14T09:09:41.899 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1000 IOs, 537495552 bytes 2024-09-14T09:09:41.980 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1100 IOs, 560036864 bytes 2024-09-14T09:09:42.174 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1200 IOs, 585563136 bytes 2024-09-14T09:09:42.306 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1300 IOs, 608580608 bytes 2024-09-14T09:09:42.592 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1400 IOs, 691688448 bytes 2024-09-14T09:09:42.723 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1500 IOs, 712323072 bytes 2024-09-14T09:09:42.951 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1600 IOs, 777428992 bytes 2024-09-14T09:09:43.153 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1700 IOs, 843266048 bytes 2024-09-14T09:09:43.323 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1800 IOs, 869639168 bytes 2024-09-14T09:09:43.624 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1900 IOs, 960992256 bytes 2024-09-14T09:09:43.754 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2000 IOs, 987189248 bytes 2024-09-14T09:09:43.996 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2100 IOs, 1052900352 bytes 2024-09-14T09:09:44.143 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2200 IOs, 1080660992 bytes 2024-09-14T09:09:44.250 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2300 IOs, 1093058560 bytes 2024-09-14T09:09:44.445 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2400 IOs, 1139726336 bytes 2024-09-14T09:09:44.605 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2500 IOs, 1172365312 bytes 2024-09-14T09:09:44.733 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2600 IOs, 1190902784 bytes 2024-09-14T09:09:44.800 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ error_code=124 2024-09-14T09:09:44.800 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set -e 2024-09-14T09:09:44.800 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T09:09:44.800 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:09:44.801 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_present cluster1 mirror image_9 present 2024-09-14T09:09:44.801 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:09:44.801 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:09:44.801 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_9 2024-09-14T09:09:44.801 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=present 2024-09-14T09:09:44.801 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:09:44.801 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T09:09:44.801 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:09:44.801 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_9 2024-09-14T09:09:44.801 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:09:44.801 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:09:44.801 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_9 2024-09-14T09:09:44.801 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_9 2024-09-14T09:09:44.802 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:09:44.847 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image_id=1539871de912 2024-09-14T09:09:44.847 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T09:09:44.847 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.1 2024-09-14T09:09:44.949 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_present cluster1 mirror image_9 present 1539871de912 2024-09-14T09:09:44.949 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:09:44.949 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:09:44.949 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_9 2024-09-14T09:09:44.949 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=present 2024-09-14T09:09:44.949 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id=1539871de912 2024-09-14T09:09:44.949 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=deleted 2024-09-14T09:09:44.949 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_image_id 2024-09-14T09:09:44.949 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_9 2024-09-14T09:09:44.949 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:09:44.949 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:09:44.949 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_9 2024-09-14T09:09:44.950 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_9 2024-09-14T09:09:44.950 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:09:44.995 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_image_id=1539871de912 2024-09-14T09:09:44.995 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n 1539871de912 2024-09-14T09:09:44.995 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -z 1539871de912 -o 1539871de912 = 1539871de912 2024-09-14T09:09:44.995 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_state=present 2024-09-14T09:09:44.995 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test present = present 2024-09-14T09:09:44.995 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:09:44.995 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ disable_mirror cluster2 mirror image_9 2024-09-14T09:09:44.995 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:09:44.995 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:09:44.995 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_9 2024-09-14T09:09:44.995 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_9 2024-09-14T09:09:45.745 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi164.stderr:2024-09-14T09:09:45.745+0000 7f0bfe298640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x560c3b7a2800 scan_remote_mirror_snapshots: failed to locate remote start snapshot: snap_id=193 2024-09-14T09:09:45.745 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi164.stderr:2024-09-14T09:09:45.745+0000 7f0bfe298640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x560c3b7a2800 scan_remote_mirror_snapshots: split-brain detected: failed to find matching non-primary snapshot in remote image: local_snap_id_start=195, local_snap_ns=[mirror state=non-primary, complete=1, mirror_peer_uuids=, primary_mirror_uuid=9ab74346-7215-4f9b-943c-26e4f257da03, primary_snap_id=c1, last_copied_object_number=128, snap_seqs={193=18446744073709551614}] 2024-09-14T09:09:46.324 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi164.stderr:2024-09-14T09:09:46.323+0000 7f3754609640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55c94be291e0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-14T09:09:46.324 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi164.stderr:2024-09-14T09:09:46.323+0000 7f375a615640 -1 rbd::mirror::ImageReplayer: 0x55c94bf16f00 [3/8cc15ce4-f134-4e45-aae8-b3e05b1aafb1] operator(): start failed: (85) Interrupted system call should be restarted 2024-09-14T09:09:50.748 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Mirroring disabled 2024-09-14T09:09:50.757 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_present cluster1 mirror image_9 deleted 2024-09-14T09:09:50.757 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:09:50.757 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:09:50.757 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_9 2024-09-14T09:09:50.757 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=deleted 2024-09-14T09:09:50.757 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:09:50.757 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T09:09:50.757 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:09:50.758 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_9 2024-09-14T09:09:50.758 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:09:50.758 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:09:50.758 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_9 2024-09-14T09:09:50.758 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_9 2024-09-14T09:09:50.758 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:09:50.798 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_9: (2) No such file or directory 2024-09-14T09:09:50.802 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image_id= 2024-09-14T09:09:50.802 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T09:09:50.802 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.1 2024-09-14T09:09:50.903 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_present cluster1 mirror image_9 deleted '' 2024-09-14T09:09:50.904 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:09:50.904 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:09:50.904 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_9 2024-09-14T09:09:50.904 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=deleted 2024-09-14T09:09:50.904 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:09:50.904 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=deleted 2024-09-14T09:09:50.904 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_image_id 2024-09-14T09:09:50.904 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_9 2024-09-14T09:09:50.904 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:09:50.904 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:09:50.904 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_9 2024-09-14T09:09:50.904 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_9 2024-09-14T09:09:50.905 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:09:50.943 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_9: (2) No such file or directory 2024-09-14T09:09:50.947 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_image_id= 2024-09-14T09:09:50.947 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:09:50.947 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test deleted = deleted 2024-09-14T09:09:50.947 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:09:50.947 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ purge_snapshots cluster2 mirror image_9 2024-09-14T09:09:50.948 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:09:50.948 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:09:50.948 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_9 2024-09-14T09:09:50.948 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap purge mirror/image_9 2024-09-14T09:09:50.995 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image_retry cluster2 mirror image_9 2024-09-14T09:09:50.995 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:09:50.995 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:09:50.995 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_9 2024-09-14T09:09:50.995 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:09:50.995 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0 2024-09-14T09:09:50.996 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_9 2024-09-14T09:09:50.996 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:09:50.996 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:09:50.996 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_9 2024-09-14T09:09:50.996 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_9 2024-09-14T09:09:51.043 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_9 2024-09-14T09:09:51.088 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 0% complete...failed. 2024-09-14T09:09:51.088 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error: image still has watchers 2024-09-14T09:09:51.088 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout. 2024-09-14T09:09:51.091 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:09:51.091 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T09:09:52.093 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_9 2024-09-14T09:09:52.093 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:09:52.093 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:09:52.093 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_9 2024-09-14T09:09:52.093 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_9 2024-09-14T09:09:52.149 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_9 2024-09-14T09:09:52.198 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 0% complete...failed. 2024-09-14T09:09:52.199 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error: image still has watchers 2024-09-14T09:09:52.199 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout. 2024-09-14T09:09:52.202 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:09:52.202 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 2 2024-09-14T09:09:54.203 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_9 2024-09-14T09:09:54.203 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:09:54.203 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:09:54.204 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_9 2024-09-14T09:09:54.204 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_9 2024-09-14T09:09:54.257 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_9 2024-09-14T09:09:54.306 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 0% complete...failed. 2024-09-14T09:09:54.306 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error: image still has watchers 2024-09-14T09:09:54.306 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout. 2024-09-14T09:09:54.309 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:09:54.309 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 4 2024-09-14T09:09:58.311 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_9 2024-09-14T09:09:58.311 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:09:58.312 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:09:58.312 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_9 2024-09-14T09:09:58.312 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_9 2024-09-14T09:09:58.365 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_9 2024-09-14T09:09:58.414 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 0% complete...failed. 2024-09-14T09:09:58.414 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error: image still has watchers 2024-09-14T09:09:58.414 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout. 2024-09-14T09:09:58.417 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:09:58.417 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 8 2024-09-14T09:10:05.078 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.753205678171253) > weight (0.25) 2024-09-14T09:10:05.078 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.8164312203707595) > weight (0.25) 2024-09-14T09:10:05.078 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.7483295805715897) > weight (0.25) 2024-09-14T09:10:05.078 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.3 with skip (0.6058042405985208) > weight (0.25) 2024-09-14T09:10:05.079 INFO:tasks.rbd_mirror_thrash:waiting for 97 secs before thrashing 2024-09-14T09:10:06.419 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_9 2024-09-14T09:10:06.419 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:10:06.419 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:10:06.419 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_9 2024-09-14T09:10:06.419 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_9 2024-09-14T09:10:06.474 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_9 2024-09-14T09:10:06.524 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 0% complete...failed. 2024-09-14T09:10:06.524 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error: image still has watchers 2024-09-14T09:10:06.524 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout. 2024-09-14T09:10:06.527 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:10:06.527 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 16 2024-09-14T09:10:22.529 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_9 2024-09-14T09:10:22.529 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:10:22.529 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:10:22.529 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_9 2024-09-14T09:10:22.529 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_9 2024-09-14T09:10:22.583 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_9 2024-09-14T09:10:22.799 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-14T09:10:22.804 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:10:22.804 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for i in `seq 1 10` 2024-09-14T09:10:22.804 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image=image_10 2024-09-14T09:10:22.804 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ create_image cluster2 mirror image_10 512M 2024-09-14T09:10:22.804 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:10:22.804 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T09:10:22.804 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:10:22.805 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T09:10:22.805 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_10 2024-09-14T09:10:22.805 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T09:10:22.805 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local size=128 2024-09-14T09:10:22.805 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -n 512M ']' 2024-09-14T09:10:22.805 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ size=512M 2024-09-14T09:10:22.805 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ shift 2024-09-14T09:10:22.805 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering mirror/image_10 2024-09-14T09:10:22.853 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ enable_mirror cluster2 mirror image_10 2024-09-14T09:10:22.853 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:10:22.853 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:10:22.853 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_10 2024-09-14T09:10:22.853 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local mode=snapshot 2024-09-14T09:10:22.853 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_10 snapshot 2024-09-14T09:10:23.706 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Mirroring enabled 2024-09-14T09:10:23.713 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 info mirror/image_10 2024-09-14T09:10:23.753 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:rbd image 'image_10': 2024-09-14T09:10:23.753 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: size 512 MiB in 128 objects 2024-09-14T09:10:23.753 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: order 22 (4 MiB objects) 2024-09-14T09:10:23.753 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: snapshot_count: 1 2024-09-14T09:10:23.753 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: id: 1adc3f4876aa 2024-09-14T09:10:23.753 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: block_name_prefix: rbd_data.1adc3f4876aa 2024-09-14T09:10:23.753 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: format: 2 2024-09-14T09:10:23.753 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: features: layering 2024-09-14T09:10:23.754 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: op_features: 2024-09-14T09:10:23.754 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: flags: 2024-09-14T09:10:23.754 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: create_timestamp: Sat Sep 14 09:10:22 2024 2024-09-14T09:10:23.754 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: access_timestamp: Sat Sep 14 09:10:22 2024 2024-09-14T09:10:23.754 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: modify_timestamp: Sat Sep 14 09:10:22 2024 2024-09-14T09:10:23.754 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring state: enabled 2024-09-14T09:10:23.754 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring mode: snapshot 2024-09-14T09:10:23.754 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring global id: 922423d7-1134-4e73-ab56-6b387b5a6870 2024-09-14T09:10:23.754 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout: mirroring primary: true 2024-09-14T09:10:23.756 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ stress_write_image cluster2 mirror image_10 2024-09-14T09:10:23.756 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:10:23.757 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:10:23.757 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_10 2024-09-14T09:10:23.757 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T09:10:23.758 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local duration=14 2024-09-14T09:10:23.759 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set +e 2024-09-14T09:10:23.759 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ timeout 14s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_10 --debug-rbd=20 --debug-journaler=20 2024-09-14T09:10:23.972 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:100 IOs, 52023296 bytes 2024-09-14T09:10:24.184 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:200 IOs, 107524096 bytes 2024-09-14T09:10:24.361 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:300 IOs, 139084800 bytes 2024-09-14T09:10:24.489 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:400 IOs, 159324160 bytes 2024-09-14T09:10:24.673 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:500 IOs, 209286144 bytes 2024-09-14T09:10:24.880 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:600 IOs, 268817408 bytes 2024-09-14T09:10:25.092 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:700 IOs, 317913088 bytes 2024-09-14T09:10:25.381 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:800 IOs, 430206976 bytes 2024-09-14T09:10:25.563 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:900 IOs, 452487168 bytes 2024-09-14T09:10:25.825 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1000 IOs, 530804736 bytes 2024-09-14T09:10:25.953 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1100 IOs, 551963648 bytes 2024-09-14T09:10:26.065 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1200 IOs, 564635648 bytes 2024-09-14T09:10:26.161 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1300 IOs, 574620672 bytes 2024-09-14T09:10:26.402 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1400 IOs, 644027392 bytes 2024-09-14T09:10:26.608 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1500 IOs, 693476352 bytes 2024-09-14T09:10:26.776 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1600 IOs, 735589376 bytes 2024-09-14T09:10:26.923 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1700 IOs, 764393472 bytes 2024-09-14T09:10:27.102 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1800 IOs, 801225728 bytes 2024-09-14T09:10:27.260 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:1900 IOs, 828516352 bytes 2024-09-14T09:10:27.350 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2000 IOs, 834283520 bytes 2024-09-14T09:10:27.528 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2100 IOs, 878147584 bytes 2024-09-14T09:10:27.779 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2200 IOs, 971344896 bytes 2024-09-14T09:10:27.991 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2300 IOs, 1013891072 bytes 2024-09-14T09:10:28.122 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2400 IOs, 1034353664 bytes 2024-09-14T09:10:28.361 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2500 IOs, 1100132352 bytes 2024-09-14T09:10:28.739 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2600 IOs, 1222212608 bytes 2024-09-14T09:10:28.985 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2700 IOs, 1289485312 bytes 2024-09-14T09:10:29.200 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2800 IOs, 1341045760 bytes 2024-09-14T09:10:29.444 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:2900 IOs, 1413174272 bytes 2024-09-14T09:10:29.657 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3000 IOs, 1466271744 bytes 2024-09-14T09:10:29.785 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3100 IOs, 1485526016 bytes 2024-09-14T09:10:29.909 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3200 IOs, 1503666176 bytes 2024-09-14T09:10:30.062 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3300 IOs, 1536928768 bytes 2024-09-14T09:10:30.174 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3400 IOs, 1557966848 bytes 2024-09-14T09:10:30.349 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3500 IOs, 1592968192 bytes 2024-09-14T09:10:30.516 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3600 IOs, 1628527616 bytes 2024-09-14T09:10:30.609 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3700 IOs, 1636478976 bytes 2024-09-14T09:10:30.749 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3800 IOs, 1656609792 bytes 2024-09-14T09:10:30.870 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:3900 IOs, 1681162240 bytes 2024-09-14T09:10:31.017 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4000 IOs, 1704040448 bytes 2024-09-14T09:10:31.287 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4100 IOs, 1782631424 bytes 2024-09-14T09:10:31.388 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4200 IOs, 1792907264 bytes 2024-09-14T09:10:31.467 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4300 IOs, 1793510400 bytes 2024-09-14T09:10:31.686 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4400 IOs, 1866678272 bytes 2024-09-14T09:10:31.843 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4500 IOs, 1887253504 bytes 2024-09-14T09:10:32.043 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4600 IOs, 1937711104 bytes 2024-09-14T09:10:32.381 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4700 IOs, 2083162112 bytes 2024-09-14T09:10:32.577 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4800 IOs, 2099694592 bytes 2024-09-14T09:10:32.865 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:4900 IOs, 2185652224 bytes 2024-09-14T09:10:32.993 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5000 IOs, 2206295040 bytes 2024-09-14T09:10:33.127 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5100 IOs, 2238131200 bytes 2024-09-14T09:10:33.351 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5200 IOs, 2290189312 bytes 2024-09-14T09:10:33.579 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5300 IOs, 2362716160 bytes 2024-09-14T09:10:33.799 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5400 IOs, 2401743872 bytes 2024-09-14T09:10:33.970 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5500 IOs, 2452717568 bytes 2024-09-14T09:10:34.254 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5600 IOs, 2524874752 bytes 2024-09-14T09:10:34.413 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5700 IOs, 2558598144 bytes 2024-09-14T09:10:34.502 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5800 IOs, 2563591168 bytes 2024-09-14T09:10:34.624 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:5900 IOs, 2602513408 bytes 2024-09-14T09:10:34.827 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6000 IOs, 2639451136 bytes 2024-09-14T09:10:34.990 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6100 IOs, 2671387648 bytes 2024-09-14T09:10:35.219 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6200 IOs, 2733246464 bytes 2024-09-14T09:10:35.304 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6300 IOs, 2734182400 bytes 2024-09-14T09:10:35.516 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6400 IOs, 2790776832 bytes 2024-09-14T09:10:35.633 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6500 IOs, 2805413888 bytes 2024-09-14T09:10:35.741 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6600 IOs, 2817675264 bytes 2024-09-14T09:10:35.873 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6700 IOs, 2844346368 bytes 2024-09-14T09:10:36.023 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6800 IOs, 2871558144 bytes 2024-09-14T09:10:36.103 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:6900 IOs, 2872181760 bytes 2024-09-14T09:10:36.326 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:7000 IOs, 2933366784 bytes 2024-09-14T09:10:36.478 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:7100 IOs, 2976310272 bytes 2024-09-14T09:10:36.723 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:7200 IOs, 3056762880 bytes 2024-09-14T09:10:36.876 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:7300 IOs, 3075211264 bytes 2024-09-14T09:10:37.160 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:7400 IOs, 3146915840 bytes 2024-09-14T09:10:37.395 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:7500 IOs, 3217400832 bytes 2024-09-14T09:10:37.522 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:7600 IOs, 3231412224 bytes 2024-09-14T09:10:37.725 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:7700 IOs, 3285619712 bytes 2024-09-14T09:10:37.764 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ error_code=124 2024-09-14T09:10:37.764 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set -e 2024-09-14T09:10:37.764 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T09:10:37.764 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:10:37.764 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_present cluster1 mirror image_10 present 2024-09-14T09:10:37.764 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:10:37.764 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:10:37.764 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_10 2024-09-14T09:10:37.765 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=present 2024-09-14T09:10:37.765 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:10:37.765 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T09:10:37.765 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:10:37.765 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_10 2024-09-14T09:10:37.765 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:10:37.765 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:10:37.765 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_10 2024-09-14T09:10:37.765 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_10 2024-09-14T09:10:37.765 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:10:37.811 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image_id=1539705ae49e 2024-09-14T09:10:37.812 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T09:10:37.812 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.1 2024-09-14T09:10:37.913 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_present cluster1 mirror image_10 present 1539705ae49e 2024-09-14T09:10:37.913 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:10:37.913 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:10:37.914 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_10 2024-09-14T09:10:37.914 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=present 2024-09-14T09:10:37.914 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id=1539705ae49e 2024-09-14T09:10:37.914 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=deleted 2024-09-14T09:10:37.914 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_image_id 2024-09-14T09:10:37.914 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_10 2024-09-14T09:10:37.914 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:10:37.914 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:10:37.914 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_10 2024-09-14T09:10:37.914 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_10 2024-09-14T09:10:37.914 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:10:37.959 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_image_id=1539705ae49e 2024-09-14T09:10:37.960 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n 1539705ae49e 2024-09-14T09:10:37.960 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -z 1539705ae49e -o 1539705ae49e = 1539705ae49e 2024-09-14T09:10:37.960 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_state=present 2024-09-14T09:10:37.960 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test present = present 2024-09-14T09:10:37.960 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:10:37.960 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ disable_mirror cluster2 mirror image_10 2024-09-14T09:10:37.960 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:10:37.960 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:10:37.961 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_10 2024-09-14T09:10:37.961 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_10 2024-09-14T09:10:38.723 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi164.stderr:2024-09-14T09:10:38.722+0000 7f0bfe298640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x560c3e8bf000 scan_remote_mirror_snapshots: failed to locate remote start snapshot: snap_id=194 2024-09-14T09:10:38.723 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi164.stderr:2024-09-14T09:10:38.722+0000 7f0bfe298640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x560c3e8bf000 scan_remote_mirror_snapshots: split-brain detected: failed to find matching non-primary snapshot in remote image: local_snap_id_start=196, local_snap_ns=[mirror state=non-primary, complete=1, mirror_peer_uuids=, primary_mirror_uuid=9ab74346-7215-4f9b-943c-26e4f257da03, primary_snap_id=c2, last_copied_object_number=128, snap_seqs={194=18446744073709551614}] 2024-09-14T09:10:42.005 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi164.stderr:2024-09-14T09:10:42.004+0000 7f3753e08640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55c94be28d00 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-14T09:10:42.005 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi164.stderr:2024-09-14T09:10:42.004+0000 7f375a615640 -1 rbd::mirror::ImageReplayer: 0x55c94bcfa000 [3/922423d7-1134-4e73-ab56-6b387b5a6870] operator(): start failed: (85) Interrupted system call should be restarted 2024-09-14T09:10:43.210 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.47187319325507115) > weight (0.25) 2024-09-14T09:10:43.211 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.5355053700307257) > weight (0.25) 2024-09-14T09:10:43.211 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.7430740299617801) > weight (0.25) 2024-09-14T09:10:43.211 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.3 2024-09-14T09:10:43.211 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Sent signal 15 2024-09-14T09:10:43.212 INFO:tasks.rbd_mirror_thrash:waiting for 7 secs before reviving daemons 2024-09-14T09:10:43.212 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi164.stderr:2024-09-14T09:10:43.212+0000 7f8824c36640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.3 (PID: 64257) UID: 1000 2024-09-14T09:10:43.725 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:Mirroring disabled 2024-09-14T09:10:43.733 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_present cluster1 mirror image_10 deleted 2024-09-14T09:10:43.733 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:10:43.734 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:10:43.734 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_10 2024-09-14T09:10:43.734 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local state=deleted 2024-09-14T09:10:43.734 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:10:43.734 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local s 2024-09-14T09:10:43.734 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:10:43.734 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_10 2024-09-14T09:10:43.734 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:10:43.734 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:10:43.734 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_10 2024-09-14T09:10:43.734 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_10 2024-09-14T09:10:43.734 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:10:43.775 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_10: (2) No such file or directory 2024-09-14T09:10:43.779 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ image_id= 2024-09-14T09:10:43.779 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T09:10:43.779 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0.1 2024-09-14T09:10:43.880 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test_image_present cluster1 mirror image_10 deleted '' 2024-09-14T09:10:43.880 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:10:43.881 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:10:43.881 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_10 2024-09-14T09:10:43.881 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local test_state=deleted 2024-09-14T09:10:43.881 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image_id= 2024-09-14T09:10:43.881 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_state=deleted 2024-09-14T09:10:43.881 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local current_image_id 2024-09-14T09:10:43.881 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ get_image_id cluster1 mirror image_10 2024-09-14T09:10:43.881 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:10:43.881 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:10:43.881 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local image=image_10 2024-09-14T09:10:43.881 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rbd --cluster=cluster1 info mirror/image_10 2024-09-14T09:10:43.881 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T09:10:43.918 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error opening image image_10: (2) No such file or directory 2024-09-14T09:10:43.922 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ current_image_id= 2024-09-14T09:10:43.922 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n '' 2024-09-14T09:10:43.922 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test deleted = deleted 2024-09-14T09:10:43.922 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:10:43.922 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ purge_snapshots cluster2 mirror image_10 2024-09-14T09:10:43.922 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:10:43.922 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:10:43.922 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_10 2024-09-14T09:10:43.922 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster2 snap purge mirror/image_10 2024-09-14T09:10:43.969 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image_retry cluster2 mirror image_10 2024-09-14T09:10:43.969 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:10:43.969 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:10:43.970 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_10 2024-09-14T09:10:43.970 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:10:43.970 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0 2024-09-14T09:10:43.970 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_10 2024-09-14T09:10:43.970 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:10:43.970 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:10:43.970 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_10 2024-09-14T09:10:43.971 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_10 2024-09-14T09:10:44.017 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_10 2024-09-14T09:10:44.062 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 0% complete...failed. 2024-09-14T09:10:44.063 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error: image still has watchers 2024-09-14T09:10:44.063 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout. 2024-09-14T09:10:44.065 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:10:44.066 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 1 2024-09-14T09:10:45.067 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_10 2024-09-14T09:10:45.067 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:10:45.067 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:10:45.067 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_10 2024-09-14T09:10:45.068 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_10 2024-09-14T09:10:45.119 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_10 2024-09-14T09:10:45.166 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 0% complete...failed. 2024-09-14T09:10:45.166 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error: image still has watchers 2024-09-14T09:10:45.166 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout. 2024-09-14T09:10:45.169 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:10:45.169 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 2 2024-09-14T09:10:47.171 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_10 2024-09-14T09:10:47.171 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:10:47.171 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:10:47.171 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_10 2024-09-14T09:10:47.171 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_10 2024-09-14T09:10:47.226 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_10 2024-09-14T09:10:47.274 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 0% complete...failed. 2024-09-14T09:10:47.275 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error: image still has watchers 2024-09-14T09:10:47.275 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout. 2024-09-14T09:10:47.278 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:10:47.278 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 4 2024-09-14T09:10:50.213 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.3 2024-09-14T09:10:50.213 INFO:teuthology.orchestra.run:waiting for 600 2024-09-14T09:10:50.213 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.3 2024-09-14T09:10:50.213 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Restarting daemon 2024-09-14T09:10:50.214 DEBUG:teuthology.orchestra.run.smithi164:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.3 2024-09-14T09:10:50.217 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Started 2024-09-14T09:10:50.217 INFO:tasks.rbd_mirror_thrash:waiting for 104 secs before thrashing 2024-09-14T09:10:51.279 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_10 2024-09-14T09:10:51.280 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:10:51.280 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:10:51.280 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_10 2024-09-14T09:10:51.280 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_10 2024-09-14T09:10:51.328 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_10 2024-09-14T09:10:51.373 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 0% complete...failed. 2024-09-14T09:10:51.374 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error: image still has watchers 2024-09-14T09:10:51.374 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout. 2024-09-14T09:10:51.377 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:10:51.377 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 8 2024-09-14T09:10:59.378 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_10 2024-09-14T09:10:59.379 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:10:59.379 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:10:59.379 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_10 2024-09-14T09:10:59.379 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_10 2024-09-14T09:10:59.432 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_10 2024-09-14T09:10:59.480 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 0% complete...failed. 2024-09-14T09:10:59.480 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:rbd: error: image still has watchers 2024-09-14T09:10:59.480 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout. 2024-09-14T09:10:59.483 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T09:10:59.483 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 16 2024-09-14T09:11:15.485 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ remove_image cluster2 mirror image_10 2024-09-14T09:11:15.485 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:11:15.485 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:11:15.485 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local image=image_10 2024-09-14T09:11:15.485 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_10 2024-09-14T09:11:15.538 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster=cluster2 rm mirror/image_10 2024-09-14T09:11:15.754 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-14T09:11:15.757 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:11:15.757 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ testlog 'TEST: check if removed images'\'' OMAP are removed' 2024-09-14T09:11:15.758 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ tee -a /tmp/tmp.IUd9LESfOA/rbd-mirror.test.log 2024-09-14T09:11:15.758 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ date '+%F %T' 2024-09-14T09:11:15.759 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ echo 2024-09-14 09:11:15 TEST: check if removed 'images'\''' OMAP are removed 2024-09-14T09:11:15.759 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:2024-09-14 09:11:15 TEST: check if removed images' OMAP are removed 2024-09-14T09:11:15.759 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_in_omap cluster1 mirror 2024-09-14T09:11:15.759 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:11:15.759 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:11:15.760 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_omap_keys cluster1 mirror rbd_mirroring status_global 2024-09-14T09:11:15.760 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:11:15.760 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:11:15.760 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local obj_name=rbd_mirroring 2024-09-14T09:11:15.760 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local filter=status_global 2024-09-14T09:11:15.760 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2024-09-14T09:11:15.760 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0 2024-09-14T09:11:15.760 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set +e 2024-09-14T09:11:15.761 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ count_omap_keys_with_filter cluster1 mirror rbd_mirroring status_global 2024-09-14T09:11:15.761 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:11:15.761 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:11:15.762 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local obj_name=rbd_mirroring 2024-09-14T09:11:15.762 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local filter=status_global 2024-09-14T09:11:15.762 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ list_omap_keys cluster1 mirror rbd_mirroring 2024-09-14T09:11:15.762 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:11:15.762 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:11:15.762 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local obj_name=rbd_mirroring 2024-09-14T09:11:15.762 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ grep -c status_global 2024-09-14T09:11:15.762 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rados --cluster cluster1 -p mirror listomapkeys rbd_mirroring 2024-09-14T09:11:15.795 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 0 = 0 2024-09-14T09:11:15.795 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ error_code=0 2024-09-14T09:11:15.795 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set -e 2024-09-14T09:11:15.796 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' 0 -eq 0 ']' 2024-09-14T09:11:15.796 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:11:15.796 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_omap_keys cluster1 mirror rbd_mirroring image_ 2024-09-14T09:11:15.796 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:11:15.796 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:11:15.796 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local obj_name=rbd_mirroring 2024-09-14T09:11:15.796 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local filter=image_ 2024-09-14T09:11:15.796 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2024-09-14T09:11:15.796 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0 2024-09-14T09:11:15.797 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set +e 2024-09-14T09:11:15.798 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ count_omap_keys_with_filter cluster1 mirror rbd_mirroring image_ 2024-09-14T09:11:15.798 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:11:15.798 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:11:15.798 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local obj_name=rbd_mirroring 2024-09-14T09:11:15.798 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local filter=image_ 2024-09-14T09:11:15.798 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ list_omap_keys cluster1 mirror rbd_mirroring 2024-09-14T09:11:15.798 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:11:15.798 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:11:15.798 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local obj_name=rbd_mirroring 2024-09-14T09:11:15.798 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ grep -c image_ 2024-09-14T09:11:15.798 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rados --cluster cluster1 -p mirror listomapkeys rbd_mirroring 2024-09-14T09:11:15.832 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 0 = 0 2024-09-14T09:11:15.832 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ error_code=0 2024-09-14T09:11:15.832 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set -e 2024-09-14T09:11:15.832 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' 0 -eq 0 ']' 2024-09-14T09:11:15.832 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:11:15.832 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_omap_keys cluster1 mirror rbd_mirror_leader image_map 2024-09-14T09:11:15.832 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:11:15.832 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:11:15.832 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local obj_name=rbd_mirror_leader 2024-09-14T09:11:15.832 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local filter=image_map 2024-09-14T09:11:15.832 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2024-09-14T09:11:15.833 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0 2024-09-14T09:11:15.833 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set +e 2024-09-14T09:11:15.834 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ count_omap_keys_with_filter cluster1 mirror rbd_mirror_leader image_map 2024-09-14T09:11:15.834 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:11:15.834 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:11:15.834 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local obj_name=rbd_mirror_leader 2024-09-14T09:11:15.834 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local filter=image_map 2024-09-14T09:11:15.834 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ list_omap_keys cluster1 mirror rbd_mirror_leader 2024-09-14T09:11:15.834 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster1 2024-09-14T09:11:15.834 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:11:15.834 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local obj_name=rbd_mirror_leader 2024-09-14T09:11:15.834 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ grep -c image_map 2024-09-14T09:11:15.835 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rados --cluster cluster1 -p mirror listomapkeys rbd_mirror_leader 2024-09-14T09:11:15.868 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 0 = 0 2024-09-14T09:11:15.868 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ error_code=0 2024-09-14T09:11:15.868 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set -e 2024-09-14T09:11:15.868 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' 0 -eq 0 ']' 2024-09-14T09:11:15.868 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:11:15.868 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_image_in_omap cluster2 mirror 2024-09-14T09:11:15.868 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:11:15.868 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:11:15.868 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_omap_keys cluster2 mirror rbd_mirroring status_global 2024-09-14T09:11:15.868 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:11:15.868 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:11:15.869 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local obj_name=rbd_mirroring 2024-09-14T09:11:15.869 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local filter=status_global 2024-09-14T09:11:15.869 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2024-09-14T09:11:15.869 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0 2024-09-14T09:11:15.870 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set +e 2024-09-14T09:11:15.870 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ count_omap_keys_with_filter cluster2 mirror rbd_mirroring status_global 2024-09-14T09:11:15.870 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster2 2024-09-14T09:11:15.870 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:11:15.870 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local obj_name=rbd_mirroring 2024-09-14T09:11:15.870 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local filter=status_global 2024-09-14T09:11:15.870 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ list_omap_keys cluster2 mirror rbd_mirroring 2024-09-14T09:11:15.870 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster2 2024-09-14T09:11:15.871 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:11:15.871 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local obj_name=rbd_mirroring 2024-09-14T09:11:15.871 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ grep -c status_global 2024-09-14T09:11:15.871 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rados --cluster cluster2 -p mirror listomapkeys rbd_mirroring 2024-09-14T09:11:15.903 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 0 = 0 2024-09-14T09:11:15.903 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ error_code=0 2024-09-14T09:11:15.903 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set -e 2024-09-14T09:11:15.903 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' 0 -eq 0 ']' 2024-09-14T09:11:15.903 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:11:15.903 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_omap_keys cluster2 mirror rbd_mirroring image_ 2024-09-14T09:11:15.903 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:11:15.903 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:11:15.904 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local obj_name=rbd_mirroring 2024-09-14T09:11:15.904 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local filter=image_ 2024-09-14T09:11:15.904 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2024-09-14T09:11:15.904 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0 2024-09-14T09:11:15.905 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set +e 2024-09-14T09:11:15.905 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ count_omap_keys_with_filter cluster2 mirror rbd_mirroring image_ 2024-09-14T09:11:15.905 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster2 2024-09-14T09:11:15.905 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:11:15.905 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local obj_name=rbd_mirroring 2024-09-14T09:11:15.905 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local filter=image_ 2024-09-14T09:11:15.905 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ list_omap_keys cluster2 mirror rbd_mirroring 2024-09-14T09:11:15.906 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster2 2024-09-14T09:11:15.906 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:11:15.906 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local obj_name=rbd_mirroring 2024-09-14T09:11:15.906 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rados --cluster cluster2 -p mirror listomapkeys rbd_mirroring 2024-09-14T09:11:15.906 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ grep -c image_ 2024-09-14T09:11:15.938 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 0 = 0 2024-09-14T09:11:15.938 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ error_code=0 2024-09-14T09:11:15.938 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set -e 2024-09-14T09:11:15.938 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' 0 -eq 0 ']' 2024-09-14T09:11:15.938 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:11:15.938 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ wait_for_omap_keys cluster2 mirror rbd_mirror_leader image_map 2024-09-14T09:11:15.938 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:11:15.938 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local pool=mirror 2024-09-14T09:11:15.938 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local obj_name=rbd_mirror_leader 2024-09-14T09:11:15.939 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local filter=image_map 2024-09-14T09:11:15.939 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2024-09-14T09:11:15.939 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ sleep 0 2024-09-14T09:11:15.940 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set +e 2024-09-14T09:11:15.940 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ count_omap_keys_with_filter cluster2 mirror rbd_mirror_leader image_map 2024-09-14T09:11:15.940 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster2 2024-09-14T09:11:15.940 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:11:15.940 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local obj_name=rbd_mirror_leader 2024-09-14T09:11:15.940 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local filter=image_map 2024-09-14T09:11:15.940 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ list_omap_keys cluster2 mirror rbd_mirror_leader 2024-09-14T09:11:15.940 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local cluster=cluster2 2024-09-14T09:11:15.940 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local pool=mirror 2024-09-14T09:11:15.940 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ local obj_name=rbd_mirror_leader 2024-09-14T09:11:15.940 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ rados --cluster cluster2 -p mirror listomapkeys rbd_mirror_leader 2024-09-14T09:11:15.941 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ grep -c image_map 2024-09-14T09:11:15.973 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test 0 = 0 2024-09-14T09:11:15.973 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ error_code=0 2024-09-14T09:11:15.973 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set -e 2024-09-14T09:11:15.973 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' 0 -eq 0 ']' 2024-09-14T09:11:15.973 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 0 2024-09-14T09:11:15.973 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ cleanup 0 2024-09-14T09:11:15.973 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local error_code=0 2024-09-14T09:11:15.974 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ set +e 2024-09-14T09:11:15.974 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' 0 -ne 0 ']' 2024-09-14T09:11:15.974 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -z '' ']' 2024-09-14T09:11:15.974 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster instance 2024-09-14T09:11:15.974 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ CEPH_ARGS= 2024-09-14T09:11:15.974 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ ceph --cluster cluster1 osd pool rm mirror mirror --yes-i-really-really-mean-it 2024-09-14T09:11:16.608 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi164.stderr:2024-09-14T09:11:16.607+0000 7f0c052a6640 -1 rbd::mirror::image_deleter::TrashWatcher: 0x560c3acdcfc0 handle_unregister_watcher: error unregistering watcher for trash directory: (2) No such file or directory 2024-09-14T09:11:16.609 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi164.stderr:2024-09-14T09:11:16.607+0000 7f0c052a6640 -1 rbd::mirror::image_deleter::TrashWatcher: 0x560c3acdc240 handle_unregister_watcher: error unregistering watcher for trash directory: (2) No such file or directory 2024-09-14T09:11:16.609 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi164.stderr:2024-09-14T09:11:16.607+0000 7f0c052a6640 -1 rbd::mirror::PoolWatcher: 0x560c3b836d20 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2024-09-14T09:11:16.609 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi164.stderr:2024-09-14T09:11:16.607+0000 7f0c052a6640 -1 rbd::mirror::PoolWatcher: 0x560c3b8374a0 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2024-09-14T09:11:16.612 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi164.stderr:2024-09-14T09:11:16.611+0000 7f0c0bab3640 -1 rbd::mirror::InstanceWatcher: 0x560c3a7a0700 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5433 object: (2) No such file or directory 2024-09-14T09:11:16.612 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi164.stderr:2024-09-14T09:11:16.611+0000 7f0c0b2b2640 -1 rbd::mirror::InstanceWatcher: 0x560c3a7a0380 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5433 object: (2) No such file or directory 2024-09-14T09:11:16.612 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi164.stderr:2024-09-14T09:11:16.611+0000 7f0c04aa5640 -1 rbd::mirror::InstanceWatcher: 0x560c3a7a0700 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2024-09-14T09:11:16.613 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi164.stderr:2024-09-14T09:11:16.611+0000 7f0c052a6640 -1 rbd::mirror::InstanceWatcher: 0x560c3a7a0380 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2024-09-14T09:11:16.615 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi164.stderr:2024-09-14T09:11:16.614+0000 7f0c04aa5640 -1 rbd::mirror::MirrorStatusUpdater 0x560c3a7f8120 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T09:11:16.615 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi164.stderr:2024-09-14T09:11:16.614+0000 7f0c0b2b2640 -1 rbd::mirror::NamespaceReplayer: 0x560c3a7f61a0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2024-09-14T09:11:16.616 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi164.stderr:2024-09-14T09:11:16.615+0000 7f0c052a6640 -1 rbd::mirror::MirrorStatusUpdater 0x560c3a7f8000 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T09:11:16.616 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi164.stderr:2024-09-14T09:11:16.615+0000 7f0c0b2b2640 -1 rbd::mirror::NamespaceReplayer: 0x560c3a7f6000 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2024-09-14T09:11:16.617 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi164.stderr:2024-09-14T09:11:16.615+0000 7f0c04aa5640 -1 rbd::mirror::image_deleter::TrashWatcher: 0x560c3acdc000 handle_unregister_watcher: error unregistering watcher for trash directory: (2) No such file or directory 2024-09-14T09:11:16.617 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi164.stderr:2024-09-14T09:11:16.616+0000 7f0c052a6640 -1 rbd::mirror::PoolWatcher: 0x560c3988c000 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2024-09-14T09:11:16.620 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi164.stderr:2024-09-14T09:11:16.618+0000 7f0c0b2b2640 -1 rbd::mirror::LeaderWatcher: 0x560c3b5d0c00 handle_notify_lock_released: error notifying leader lock released: (2) No such file or directory 2024-09-14T09:11:16.620 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi164.stderr:2024-09-14T09:11:16.618+0000 7f0c0b2b2640 -1 rbd::mirror::LeaderWatcher: 0x560c3b5d0c00 handle_shut_down_leader_lock: error shutting down leader lock: (2) No such file or directory 2024-09-14T09:11:16.620 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi164.stderr:2024-09-14T09:11:16.619+0000 7f0c0bab3640 -1 rbd::mirror::LeaderWatcher: 0x560c3b5d0c00 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2024-09-14T09:11:16.620 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi164.stderr:2024-09-14T09:11:16.619+0000 7f0c0bab3640 -1 rbd::mirror::InstanceWatcher: 0x560c3a7a0000 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5433 object: (2) No such file or directory 2024-09-14T09:11:16.621 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi164.stderr:2024-09-14T09:11:16.620+0000 7f0c052a6640 -1 rbd::mirror::InstanceWatcher: 0x560c3a7a0000 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2024-09-14T09:11:16.623 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi164.stderr:2024-09-14T09:11:16.622+0000 7f0c04aa5640 -1 rbd::mirror::MirrorStatusUpdater 0x560c3b3cf440 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T09:11:16.623 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi164.stderr:2024-09-14T09:11:16.622+0000 7f0c0b2b2640 -1 rbd::mirror::NamespaceReplayer: 0x560c3b2de000 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2024-09-14T09:11:16.645 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:pool 'mirror' does not exist 2024-09-14T09:11:16.657 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ CEPH_ARGS= 2024-09-14T09:11:16.658 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ ceph --cluster cluster2 osd pool rm mirror mirror --yes-i-really-really-mean-it 2024-09-14T09:11:17.841 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:pool 'mirror' does not exist 2024-09-14T09:11:17.852 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ CEPH_ARGS= 2024-09-14T09:11:17.853 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ ceph --cluster cluster1 osd pool rm mirror_parent mirror_parent --yes-i-really-really-mean-it 2024-09-14T09:11:18.410 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi164.stderr:2024-09-14T09:11:18.408+0000 7f375a615640 -1 rbd::mirror::LeaderWatcher: 0x55c94c0f8f00 handle_notify_heartbeat: error notifying heartbeat: (2) No such file or directory, releasing leader 2024-09-14T09:11:18.410 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi164.stderr:2024-09-14T09:11:18.408+0000 7f3754609640 -1 rbd::mirror::image_deleter::TrashWatcher: 0x55c94b959680 handle_unregister_watcher: error unregistering watcher for trash directory: (2) No such file or directory 2024-09-14T09:11:18.410 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi164.stderr:2024-09-14T09:11:18.408+0000 7f3754609640 -1 rbd::mirror::image_deleter::TrashWatcher: 0x55c94b958480 handle_unregister_watcher: error unregistering watcher for trash directory: (2) No such file or directory 2024-09-14T09:11:18.410 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi164.stderr:2024-09-14T09:11:18.408+0000 7f3753e08640 -1 rbd::mirror::image_deleter::TrashWatcher: 0x55c94c39e240 handle_unregister_watcher: error unregistering watcher for trash directory: (2) No such file or directory 2024-09-14T09:11:18.410 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi164.stderr:2024-09-14T09:11:18.408+0000 7f3754609640 -1 rbd::mirror::PoolWatcher: 0x55c94a5eb860 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2024-09-14T09:11:18.410 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi164.stderr:2024-09-14T09:11:18.408+0000 7f3753e08640 -1 rbd::mirror::PoolWatcher: 0x55c94bb221e0 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2024-09-14T09:11:18.410 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi164.stderr:2024-09-14T09:11:18.408+0000 7f3753e08640 -1 rbd::mirror::PoolWatcher: 0x55c94b8aa1e0 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2024-09-14T09:11:18.410 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi164.stderr:2024-09-14T09:11:18.408+0000 7f374d5fb640 -1 rbd::mirror::PoolWatcher: 0x55c94bb22b40 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2024-09-14T09:11:18.410 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi164.stderr:2024-09-14T09:11:18.408+0000 7f374d5fb640 -1 rbd::mirror::PoolWatcher: 0x55c94a5eb2c0 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2024-09-14T09:11:18.410 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi164.stderr:2024-09-14T09:11:18.408+0000 7f374ddfc640 -1 rbd::mirror::PoolWatcher: 0x55c94bb22000 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2024-09-14T09:11:18.411 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi164.stderr:2024-09-14T09:11:18.408+0000 7f375a615640 -1 rbd::mirror::LeaderWatcher: 0x55c94c0f8f00 handle_notify_lock_released: error notifying leader lock released: (2) No such file or directory 2024-09-14T09:11:18.411 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi164.stderr:2024-09-14T09:11:18.408+0000 7f375a615640 -1 rbd::mirror::LeaderWatcher: 0x55c94c0f8f00 handle_release_leader_lock: error releasing lock: (2) No such file or directory 2024-09-14T09:11:18.647 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:pool 'mirror_parent' does not exist 2024-09-14T09:11:18.658 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ CEPH_ARGS= 2024-09-14T09:11:18.658 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ ceph --cluster cluster2 osd pool rm mirror_parent mirror_parent --yes-i-really-really-mean-it 2024-09-14T09:11:19.336 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi164.stderr:2024-09-14T09:11:19.334+0000 7f3e77968640 -1 rbd::mirror::RemotePoolPoller: 0x5648fd7025b0 handle_mirror_peer_ping: failed to ping remote mirror peer: (2) No such file or directory 2024-09-14T09:11:19.336 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi164.stderr:2024-09-14T09:11:19.334+0000 7f3e78169640 -1 rbd::mirror::RemotePoolPoller: 0x5648fd7025b0 handle_mirror_peer_list: failed to retrieve mirror peers: (2) No such file or directory 2024-09-14T09:11:19.336 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi164.stderr:2024-09-14T09:11:19.334+0000 7f3e78169640 -1 rbd::mirror::PoolReplayer: 0x5648fd715b00 handle_remote_pool_meta_updated: remote pool metadata updated unexpectedly 2024-09-14T09:11:19.337 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi164.stderr:2024-09-14T09:11:19.335+0000 7f3e84982640 -1 rbd::mirror::InstanceWatcher: 0x5648fe642700 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5222 object: (2) No such file or directory 2024-09-14T09:11:19.338 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi164.stderr:2024-09-14T09:11:19.335+0000 7f3e84982640 -1 rbd::mirror::InstanceWatcher: 0x5648fe642380 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5222 object: (2) No such file or directory 2024-09-14T09:11:19.338 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi164.stderr:2024-09-14T09:11:19.335+0000 7f3e7e175640 -1 rbd::mirror::InstanceWatcher: 0x5648fe642700 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2024-09-14T09:11:19.338 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi164.stderr:2024-09-14T09:11:19.335+0000 7f3e7e175640 -1 rbd::mirror::InstanceWatcher: 0x5648fe642380 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2024-09-14T09:11:19.338 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi164.stderr:2024-09-14T09:11:19.336+0000 7f3e78169640 -1 rbd::mirror::MirrorStatusUpdater 0x5648fe6a87e0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T09:11:19.338 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi164.stderr:2024-09-14T09:11:19.336+0000 7f3e78169640 -1 rbd::mirror::MirrorStatusUpdater 0x5648fe6a8360 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T09:11:19.338 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi164.stderr:2024-09-14T09:11:19.336+0000 7f3e84982640 -1 rbd::mirror::NamespaceReplayer: 0x5648fe6a6000 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2024-09-14T09:11:19.338 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi164.stderr:2024-09-14T09:11:19.336+0000 7f3e84982640 -1 rbd::mirror::NamespaceReplayer: 0x5648fe6a61a0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2024-09-14T09:11:19.338 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi164.stderr:2024-09-14T09:11:19.336+0000 7f3e7e976640 -1 rbd::mirror::MirrorStatusUpdater 0x5648fe6a8000 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T09:11:19.338 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi164.stderr:2024-09-14T09:11:19.336+0000 7f3e7e976640 -1 rbd::mirror::MirrorStatusUpdater 0x5648fe6a8120 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T09:11:19.338 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi164.stderr:2024-09-14T09:11:19.336+0000 7f3e85183640 -1 rbd::mirror::NamespaceReplayer: 0x5648fe6a6000 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2024-09-14T09:11:19.338 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi164.stderr:2024-09-14T09:11:19.336+0000 7f3e85183640 -1 rbd::mirror::NamespaceReplayer: 0x5648fe6a61a0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2024-09-14T09:11:19.780 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi164.stderr:2024-09-14T09:11:19.778+0000 7f3e6a14d640 -1 rbd::mirror::RemotePoolPoller: 0x5648fd703040 handle_mirror_peer_ping: failed to ping remote mirror peer: (2) No such file or directory 2024-09-14T09:11:19.780 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi164.stderr:2024-09-14T09:11:19.779+0000 7f3e6a94e640 -1 rbd::mirror::RemotePoolPoller: 0x5648fd703040 handle_mirror_peer_list: failed to retrieve mirror peers: (2) No such file or directory 2024-09-14T09:11:19.780 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi164.stderr:2024-09-14T09:11:19.779+0000 7f3e6a94e640 -1 rbd::mirror::PoolReplayer: 0x5648ff45ef00 handle_remote_pool_meta_updated: remote pool metadata updated unexpectedly 2024-09-14T09:11:19.800 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:pool 'mirror_parent' does not exist 2024-09-14T09:11:19.811 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for cluster in "${CLUSTER1}" "${CLUSTER2}" 2024-09-14T09:11:19.812 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ stop_mirrors cluster1 2024-09-14T09:11:19.812 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1 2024-09-14T09:11:19.812 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local sig= 2024-09-14T09:11:19.812 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ seq 0 3 2024-09-14T09:11:19.817 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T09:11:19.817 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ stop_mirror cluster1:0 '' 2024-09-14T09:11:19.817 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1:0 2024-09-14T09:11:19.817 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local sig= 2024-09-14T09:11:19.817 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n 1 2024-09-14T09:11:19.817 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 2024-09-14T09:11:19.817 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T09:11:19.817 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ stop_mirror cluster1:1 '' 2024-09-14T09:11:19.817 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1:1 2024-09-14T09:11:19.817 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local sig= 2024-09-14T09:11:19.817 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n 1 2024-09-14T09:11:19.817 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 2024-09-14T09:11:19.817 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T09:11:19.818 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ stop_mirror cluster1:2 '' 2024-09-14T09:11:19.818 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1:2 2024-09-14T09:11:19.818 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local sig= 2024-09-14T09:11:19.818 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n 1 2024-09-14T09:11:19.818 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 2024-09-14T09:11:19.818 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T09:11:19.818 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ stop_mirror cluster1:3 '' 2024-09-14T09:11:19.818 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster1:3 2024-09-14T09:11:19.818 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local sig= 2024-09-14T09:11:19.818 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n 1 2024-09-14T09:11:19.818 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 2024-09-14T09:11:19.818 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for cluster in "${CLUSTER1}" "${CLUSTER2}" 2024-09-14T09:11:19.818 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ stop_mirrors cluster2 2024-09-14T09:11:19.818 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2 2024-09-14T09:11:19.819 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local sig= 2024-09-14T09:11:19.819 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:++ seq 0 3 2024-09-14T09:11:19.819 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T09:11:19.819 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ stop_mirror cluster2:0 '' 2024-09-14T09:11:19.819 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2:0 2024-09-14T09:11:19.819 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local sig= 2024-09-14T09:11:19.819 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n 1 2024-09-14T09:11:19.819 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 2024-09-14T09:11:19.819 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T09:11:19.819 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ stop_mirror cluster2:1 '' 2024-09-14T09:11:19.819 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2:1 2024-09-14T09:11:19.819 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local sig= 2024-09-14T09:11:19.820 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n 1 2024-09-14T09:11:19.820 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 2024-09-14T09:11:19.820 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T09:11:19.820 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ stop_mirror cluster2:2 '' 2024-09-14T09:11:19.820 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2:2 2024-09-14T09:11:19.820 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local sig= 2024-09-14T09:11:19.820 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n 1 2024-09-14T09:11:19.820 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 2024-09-14T09:11:19.820 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T09:11:19.820 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ stop_mirror cluster2:3 '' 2024-09-14T09:11:19.820 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local cluster=cluster2:3 2024-09-14T09:11:19.820 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ local sig= 2024-09-14T09:11:19.821 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test -n 1 2024-09-14T09:11:19.821 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ return 2024-09-14T09:11:19.821 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' -z 1 ']' 2024-09-14T09:11:19.821 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ test '' = /tmp/tmp.IUd9LESfOA 2024-09-14T09:11:19.821 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rm -Rf /tmp/tmp.IUd9LESfOA 2024-09-14T09:11:19.821 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ '[' 0 -eq 0 ']' 2024-09-14T09:11:19.821 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ echo OK 2024-09-14T09:11:19.821 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stdout:OK 2024-09-14T09:11:19.822 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ exit 0 2024-09-14T09:11:19.823 INFO:teuthology.orchestra.run:Running command with timeout 3600 2024-09-14T09:11:19.823 DEBUG:teuthology.orchestra.run.smithi164:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror/tmp 2024-09-14T09:11:19.890 INFO:tasks.workunit:Stopping ['rbd/rbd_mirror_stress.sh'] on cluster1.client.mirror... 2024-09-14T09:11:19.890 DEBUG:teuthology.orchestra.run.smithi164:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.cluster1.client.mirror /home/ubuntu/cephtest/clone.cluster1.client.mirror 2024-09-14T09:11:20.035 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi164.stderr:2024-09-14T09:11:20.033+0000 7f3e84982640 -1 rbd::mirror::LeaderWatcher: 0x5648ff45ec00 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2024-09-14T09:11:20.035 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi164.stderr:2024-09-14T09:11:20.034+0000 7f3e84982640 -1 rbd::mirror::InstanceWatcher: 0x5648fe642000 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5222 object: (2) No such file or directory 2024-09-14T09:11:20.036 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi164.stderr:2024-09-14T09:11:20.034+0000 7f3e7e175640 -1 rbd::mirror::InstanceWatcher: 0x5648fe642000 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2024-09-14T09:11:20.036 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi164.stderr:2024-09-14T09:11:20.035+0000 7f3e77968640 -1 rbd::mirror::MirrorStatusUpdater 0x5648ff2a06c0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T09:11:20.036 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi164.stderr:2024-09-14T09:11:20.035+0000 7f3e84982640 -1 rbd::mirror::NamespaceReplayer: 0x5648ff13c000 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2024-09-14T09:11:20.037 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi164.stderr:2024-09-14T09:11:20.035+0000 7f3e7e976640 -1 rbd::mirror::MirrorStatusUpdater 0x5648ff21d440 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T09:11:20.037 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi164.stderr:2024-09-14T09:11:20.035+0000 7f3e85183640 -1 rbd::mirror::NamespaceReplayer: 0x5648ff13c000 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2024-09-14T09:11:20.040 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi164.stderr:2024-09-14T09:11:20.039+0000 7f3e84982640 -1 rbd::mirror::LeaderWatcher: 0x5648fe8ad200 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2024-09-14T09:11:20.041 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi164.stderr:2024-09-14T09:11:20.040+0000 7f3e84982640 -1 rbd::mirror::InstanceWatcher: 0x5648feb56000 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5225 object: (2) No such file or directory 2024-09-14T09:11:20.041 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi164.stderr:2024-09-14T09:11:20.040+0000 7f3e7115b640 -1 rbd::mirror::InstanceWatcher: 0x5648feb56000 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2024-09-14T09:11:20.042 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi164.stderr:2024-09-14T09:11:20.040+0000 7f3e7095a640 -1 rbd::mirror::MirrorStatusUpdater 0x5648fe84dd40 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T09:11:20.042 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi164.stderr:2024-09-14T09:11:20.040+0000 7f3e84982640 -1 rbd::mirror::NamespaceReplayer: 0x5648fd6f9520 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2024-09-14T09:11:20.229 DEBUG:teuthology.parallel:result is None 2024-09-14T09:11:20.230 DEBUG:teuthology.orchestra.run.smithi164:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror 2024-09-14T09:11:20.254 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror 2024-09-14T09:11:20.255 DEBUG:teuthology.orchestra.run.smithi164:> rmdir -- /home/ubuntu/cephtest/mnt.cluster1.mirror 2024-09-14T09:11:20.335 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi164.stderr:2024-09-14T09:11:20.334+0000 7fbea8441640 -1 rbd::mirror::RemotePoolPoller: 0x55dad4829790 handle_mirror_peer_ping: failed to ping remote mirror peer: (2) No such file or directory 2024-09-14T09:11:20.335 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi164.stderr:2024-09-14T09:11:20.334+0000 7fbea7c40640 -1 rbd::mirror::RemotePoolPoller: 0x55dad4829790 handle_mirror_peer_list: failed to retrieve mirror peers: (2) No such file or directory 2024-09-14T09:11:20.335 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi164.stderr:2024-09-14T09:11:20.334+0000 7fbea7c40640 -1 rbd::mirror::PoolReplayer: 0x55dad483bb00 handle_remote_pool_meta_updated: remote pool metadata updated unexpectedly 2024-09-14T09:11:20.337 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi164.stderr:2024-09-14T09:11:20.335+0000 7fbeb4c5a640 -1 rbd::mirror::InstanceWatcher: 0x55dad59e0380 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.6194 object: (2) No such file or directory 2024-09-14T09:11:20.337 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi164.stderr:2024-09-14T09:11:20.335+0000 7fbeb4c5a640 -1 rbd::mirror::InstanceWatcher: 0x55dad5ed6000 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.6194 object: (2) No such file or directory 2024-09-14T09:11:20.337 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi164.stderr:2024-09-14T09:11:20.335+0000 7fbeaec4e640 -1 rbd::mirror::InstanceWatcher: 0x55dad59e0380 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2024-09-14T09:11:20.337 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi164.stderr:2024-09-14T09:11:20.335+0000 7fbeaec4e640 -1 rbd::mirror::InstanceWatcher: 0x55dad5ed6000 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2024-09-14T09:11:20.337 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi164.stderr:2024-09-14T09:11:20.335+0000 7fbea8441640 -1 rbd::mirror::MirrorStatusUpdater 0x55dad5a325a0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T09:11:20.337 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi164.stderr:2024-09-14T09:11:20.335+0000 7fbea8441640 -1 rbd::mirror::MirrorStatusUpdater 0x55dad5a32360 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T09:11:20.337 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi164.stderr:2024-09-14T09:11:20.335+0000 7fbeb4c5a640 -1 rbd::mirror::NamespaceReplayer: 0x55dad5a30000 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2024-09-14T09:11:20.337 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi164.stderr:2024-09-14T09:11:20.335+0000 7fbeb4c5a640 -1 rbd::mirror::NamespaceReplayer: 0x55dad5a301a0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2024-09-14T09:11:20.337 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi164.stderr:2024-09-14T09:11:20.335+0000 7fbeae44d640 -1 rbd::mirror::MirrorStatusUpdater 0x55dad5a32000 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T09:11:20.337 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi164.stderr:2024-09-14T09:11:20.335+0000 7fbeae44d640 -1 rbd::mirror::MirrorStatusUpdater 0x55dad5a32120 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T09:11:20.338 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi164.stderr:2024-09-14T09:11:20.335+0000 7fbeb545b640 -1 rbd::mirror::NamespaceReplayer: 0x55dad5a30000 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2024-09-14T09:11:20.338 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi164.stderr:2024-09-14T09:11:20.335+0000 7fbeb545b640 -1 rbd::mirror::NamespaceReplayer: 0x55dad5a301a0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2024-09-14T09:11:20.388 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi164.stderr:2024-09-14T09:11:20.386+0000 7faecbb1d640 -1 rbd::mirror::InstanceWatcher: 0x55b81861e380 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.6029 object: (2) No such file or directory 2024-09-14T09:11:20.388 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi164.stderr:2024-09-14T09:11:20.386+0000 7faecbb1d640 -1 rbd::mirror::InstanceWatcher: 0x55b81861e700 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.6029 object: (2) No such file or directory 2024-09-14T09:11:20.388 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi164.stderr:2024-09-14T09:11:20.386+0000 7faec5b11640 -1 rbd::mirror::InstanceWatcher: 0x55b81861e380 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2024-09-14T09:11:20.388 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi164.stderr:2024-09-14T09:11:20.386+0000 7faec5b11640 -1 rbd::mirror::InstanceWatcher: 0x55b81861e700 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2024-09-14T09:11:20.388 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi164.stderr:2024-09-14T09:11:20.386+0000 7faebeb03640 -1 rbd::mirror::MirrorStatusUpdater 0x55b8177ae5a0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T09:11:20.388 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi164.stderr:2024-09-14T09:11:20.386+0000 7faebeb03640 -1 rbd::mirror::MirrorStatusUpdater 0x55b8177ae360 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T09:11:20.388 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi164.stderr:2024-09-14T09:11:20.386+0000 7faecbb1d640 -1 rbd::mirror::NamespaceReplayer: 0x55b81779c000 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2024-09-14T09:11:20.388 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi164.stderr:2024-09-14T09:11:20.386+0000 7faecbb1d640 -1 rbd::mirror::NamespaceReplayer: 0x55b81779c1a0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2024-09-14T09:11:20.388 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi164.stderr:2024-09-14T09:11:20.387+0000 7faec5310640 -1 rbd::mirror::MirrorStatusUpdater 0x55b8177ae000 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T09:11:20.388 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi164.stderr:2024-09-14T09:11:20.387+0000 7faec5310640 -1 rbd::mirror::MirrorStatusUpdater 0x55b8177ae120 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T09:11:20.388 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi164.stderr:2024-09-14T09:11:20.387+0000 7faecc31e640 -1 rbd::mirror::NamespaceReplayer: 0x55b81779c000 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2024-09-14T09:11:20.389 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi164.stderr:2024-09-14T09:11:20.387+0000 7faecc31e640 -1 rbd::mirror::NamespaceReplayer: 0x55b81779c1a0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2024-09-14T09:11:20.389 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi164.stderr:2024-09-14T09:11:20.387+0000 7faebeb03640 -1 rbd::mirror::RemotePoolPoller: 0x55b8168605b0 handle_mirror_peer_ping: failed to ping remote mirror peer: (2) No such file or directory 2024-09-14T09:11:20.389 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi164.stderr:2024-09-14T09:11:20.387+0000 7faebf304640 -1 rbd::mirror::RemotePoolPoller: 0x55b8168605b0 handle_mirror_peer_list: failed to retrieve mirror peers: (2) No such file or directory 2024-09-14T09:11:20.390 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi164.stderr:2024-09-14T09:11:20.387+0000 7faebf304640 -1 rbd::mirror::PoolReplayer: 0x55b816873b00 handle_remote_pool_meta_updated: remote pool metadata updated unexpectedly 2024-09-14T09:11:20.396 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi164.stderr:2024-09-14T09:11:20.394+0000 7fbe9a425640 -1 rbd::mirror::RemotePoolPoller: 0x55dad48284e0 handle_mirror_peer_ping: failed to ping remote mirror peer: (2) No such file or directory 2024-09-14T09:11:20.396 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi164.stderr:2024-09-14T09:11:20.394+0000 7fbe9ac26640 -1 rbd::mirror::RemotePoolPoller: 0x55dad48284e0 handle_mirror_peer_list: failed to retrieve mirror peers: (2) No such file or directory 2024-09-14T09:11:20.397 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi164.stderr:2024-09-14T09:11:20.394+0000 7fbe9ac26640 -1 rbd::mirror::PoolReplayer: 0x55dad5944f00 handle_remote_pool_meta_updated: remote pool metadata updated unexpectedly 2024-09-14T09:11:20.400 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror 2024-09-14T09:11:20.400 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror-thrash 2024-09-14T09:11:20.412 INFO:tasks.rbd_mirror_thrash:joining rbd_mirror_thrash 2024-09-14T09:11:20.413 INFO:tasks.rbd_mirror_thrash:stat['kill'] = 12 2024-09-14T09:11:20.413 INFO:tasks.rbd_mirror_thrash:done joining 2024-09-14T09:11:20.413 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror-thrash 2024-09-14T09:11:20.421 INFO:tasks.rbd_mirror_thrash:joining rbd_mirror_thrash 2024-09-14T09:11:20.422 INFO:tasks.rbd_mirror_thrash:stat['kill'] = 18 2024-09-14T09:11:20.422 INFO:tasks.rbd_mirror_thrash:done joining 2024-09-14T09:11:20.422 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror 2024-09-14T09:11:20.431 DEBUG:tasks.rbd_mirror.cluster2.client.mirror.3:waiting for process to exit 2024-09-14T09:11:20.431 INFO:teuthology.orchestra.run:waiting for 300 2024-09-14T09:11:20.432 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi164.stderr:2024-09-14T09:11:20.413+0000 7fbeb4c5a640 -1 rbd::mirror::LeaderWatcher: 0x55dad5944c00 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2024-09-14T09:11:20.432 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi164.stderr:2024-09-14T09:11:20.413+0000 7fbeb4c5a640 -1 rbd::mirror::InstanceWatcher: 0x55dad59e0000 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.6194 object: (2) No such file or directory 2024-09-14T09:11:20.432 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi164.stderr:2024-09-14T09:11:20.414+0000 7fbeaec4e640 -1 rbd::mirror::InstanceWatcher: 0x55dad59e0000 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2024-09-14T09:11:20.432 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi164.stderr:2024-09-14T09:11:20.414+0000 7fbea7c40640 -1 rbd::mirror::MirrorStatusUpdater 0x55dad57507e0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T09:11:20.432 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi164.stderr:2024-09-14T09:11:20.414+0000 7fbeb545b640 -1 rbd::mirror::NamespaceReplayer: 0x55dad481f520 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2024-09-14T09:11:20.432 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi164.stderr:2024-09-14T09:11:20.414+0000 7fbeae44d640 -1 rbd::mirror::MirrorStatusUpdater 0x55dad574f320 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T09:11:20.432 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi164.stderr:2024-09-14T09:11:20.414+0000 7fbeb4c5a640 -1 rbd::mirror::NamespaceReplayer: 0x55dad481f520 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2024-09-14T09:11:20.433 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi164.stderr:2024-09-14T09:11:20.417+0000 7fbea0c32640 -1 rbd::mirror::InstanceWatcher: 0x55dad59e1c00 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2024-09-14T09:11:20.433 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi164.stderr:2024-09-14T09:11:20.417+0000 7fbe9a425640 -1 rbd::mirror::MirrorStatusUpdater 0x55dad5c287e0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T09:11:20.433 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi164.stderr:2024-09-14T09:11:20.417+0000 7fbeb4c5a640 -1 rbd::mirror::NamespaceReplayer: 0x55dad5ea4000 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2024-09-14T09:11:20.433 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi164.stderr:2024-09-14T09:11:20.431+0000 7f3e85984640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.3 (PID: 49027) UID: 1000 2024-09-14T09:11:20.437 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi164.stderr:2024-09-14T09:11:20.436+0000 7faeb1ae9640 -1 rbd::mirror::RemotePoolPoller: 0x55b816860680 handle_mirror_peer_ping: failed to ping remote mirror peer: (2) No such file or directory 2024-09-14T09:11:20.437 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi164.stderr:2024-09-14T09:11:20.437+0000 7faeb12e8640 -1 rbd::mirror::RemotePoolPoller: 0x55b816860680 handle_mirror_peer_list: failed to retrieve mirror peers: (2) No such file or directory 2024-09-14T09:11:20.437 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi164.stderr:2024-09-14T09:11:20.437+0000 7faeb12e8640 -1 rbd::mirror::PoolReplayer: 0x55b818586f00 handle_remote_pool_meta_updated: remote pool metadata updated unexpectedly 2024-09-14T09:11:20.441 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Stopped 2024-09-14T09:11:20.441 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror 2024-09-14T09:11:20.451 DEBUG:tasks.rbd_mirror.cluster2.client.mirror.2:waiting for process to exit 2024-09-14T09:11:20.451 INFO:teuthology.orchestra.run:waiting for 300 2024-09-14T09:11:20.484 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi164.stderr:2024-09-14T09:11:20.483+0000 7f375b617640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2 (PID: 36897) UID: 1000 2024-09-14T09:11:20.485 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi164.stderr:2024-09-14T09:11:20.484+0000 7f3746dee640 -1 rbd::mirror::image_deleter::TrashWatcher: 0x55c94bd5eb40 handle_unregister_watcher: error unregistering watcher for trash directory: (2) No such file or directory 2024-09-14T09:11:20.485 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi164.stderr:2024-09-14T09:11:20.484+0000 7f375ae16640 -1 rbd::mirror::InstanceWatcher: 0x55c94b86bc00 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.4276 object: (2) No such file or directory 2024-09-14T09:11:20.485 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi164.stderr:2024-09-14T09:11:20.484+0000 7f375a615640 -1 rbd::mirror::InstanceWatcher: 0x55c94b86b880 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.4276 object: (2) No such file or directory 2024-09-14T09:11:20.485 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi164.stderr:2024-09-14T09:11:20.484+0000 7f37465ed640 -1 rbd::mirror::PoolWatcher: 0x55c94ba57860 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2024-09-14T09:11:20.485 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi164.stderr:2024-09-14T09:11:20.484+0000 7f3754609640 -1 rbd::mirror::InstanceWatcher: 0x55c94b86bc00 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2024-09-14T09:11:20.485 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi164.stderr:2024-09-14T09:11:20.484+0000 7f3754609640 -1 rbd::mirror::InstanceWatcher: 0x55c94b86b880 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2024-09-14T09:11:20.486 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi164.stderr:2024-09-14T09:11:20.484+0000 7f37405e1640 -1 rbd::mirror::PoolWatcher: 0x55c94bb22d20 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2024-09-14T09:11:20.486 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi164.stderr:2024-09-14T09:11:20.485+0000 7f374ddfc640 -1 rbd::mirror::MirrorStatusUpdater 0x55c94b972b40 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T09:11:20.486 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi164.stderr:2024-09-14T09:11:20.485+0000 7f374ddfc640 -1 rbd::mirror::MirrorStatusUpdater 0x55c94b972a20 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T09:11:20.486 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi164.stderr:2024-09-14T09:11:20.485+0000 7f375ae16640 -1 rbd::mirror::NamespaceReplayer: 0x55c94ba12000 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2024-09-14T09:11:20.486 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi164.stderr:2024-09-14T09:11:20.485+0000 7f375ae16640 -1 rbd::mirror::NamespaceReplayer: 0x55c94ba121a0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2024-09-14T09:11:20.486 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi164.stderr:2024-09-14T09:11:20.485+0000 7f3753e08640 -1 rbd::mirror::MirrorStatusUpdater 0x55c94b972480 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T09:11:20.486 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi164.stderr:2024-09-14T09:11:20.485+0000 7f3753e08640 -1 rbd::mirror::MirrorStatusUpdater 0x55c94b9725a0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T09:11:20.486 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi164.stderr:2024-09-14T09:11:20.485+0000 7f375a615640 -1 rbd::mirror::NamespaceReplayer: 0x55c94ba12000 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2024-09-14T09:11:20.486 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi164.stderr:2024-09-14T09:11:20.485+0000 7f375a615640 -1 rbd::mirror::NamespaceReplayer: 0x55c94ba121a0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2024-09-14T09:11:20.487 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi164.stderr:2024-09-14T09:11:20.486+0000 7f375a615640 -1 rbd::mirror::LeaderWatcher: 0x55c94c39d500 handle_notify_lock_released: error notifying leader lock released: (2) No such file or directory 2024-09-14T09:11:20.487 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi164.stderr:2024-09-14T09:11:20.486+0000 7f375a615640 -1 rbd::mirror::LeaderWatcher: 0x55c94c39d500 handle_shut_down_leader_lock: error shutting down leader lock: (2) No such file or directory 2024-09-14T09:11:20.488 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi164.stderr:2024-09-14T09:11:20.486+0000 7f375ae16640 -1 rbd::mirror::LeaderWatcher: 0x55c94c39d500 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2024-09-14T09:11:20.488 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi164.stderr:2024-09-14T09:11:20.487+0000 7f375ae16640 -1 rbd::mirror::InstanceWatcher: 0x55c94b86b500 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.4282 object: (2) No such file or directory 2024-09-14T09:11:20.488 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi164.stderr:2024-09-14T09:11:20.487+0000 7f37465ed640 -1 rbd::mirror::InstanceWatcher: 0x55c94b86b500 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2024-09-14T09:11:20.488 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi164.stderr:2024-09-14T09:11:20.487+0000 7f373fde0640 -1 rbd::mirror::MirrorStatusUpdater 0x55c94b9727e0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T09:11:20.488 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi164.stderr:2024-09-14T09:11:20.487+0000 7f375a615640 -1 rbd::mirror::NamespaceReplayer: 0x55c94b69a000 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2024-09-14T09:11:20.488 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi164.stderr:2024-09-14T09:11:20.488+0000 7f3746dee640 -1 rbd::mirror::MirrorStatusUpdater 0x55c94b7ccfc0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T09:11:20.488 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi164.stderr:2024-09-14T09:11:20.488+0000 7f375ae16640 -1 rbd::mirror::NamespaceReplayer: 0x55c94b69a000 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2024-09-14T09:11:20.490 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi164.stderr:2024-09-14T09:11:20.490+0000 7f375a615640 -1 rbd::mirror::LeaderWatcher: 0x55c94c0f8f00 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2024-09-14T09:11:20.490 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi164.stderr:2024-09-14T09:11:20.490+0000 7f375a615640 -1 rbd::mirror::InstanceWatcher: 0x55c94c1dee00 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.4276 object: (2) No such file or directory 2024-09-14T09:11:20.490 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi164.stderr:2024-09-14T09:11:20.490+0000 7f3754609640 -1 rbd::mirror::InstanceWatcher: 0x55c94c1dee00 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2024-09-14T09:11:20.491 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi164.stderr:2024-09-14T09:11:20.491+0000 7f374d5fb640 -1 rbd::mirror::MirrorStatusUpdater 0x55c94c1a45a0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T09:11:20.491 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi164.stderr:2024-09-14T09:11:20.491+0000 7f375ae16640 -1 rbd::mirror::NamespaceReplayer: 0x55c94b4904e0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2024-09-14T09:11:20.491 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi164.stderr:2024-09-14T09:11:20.491+0000 7f3753e08640 -1 rbd::mirror::MirrorStatusUpdater 0x55c94c191440 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T09:11:20.491 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi164.stderr:2024-09-14T09:11:20.491+0000 7f375a615640 -1 rbd::mirror::NamespaceReplayer: 0x55c94b4904e0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2024-09-14T09:11:20.503 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Stopped 2024-09-14T09:11:20.503 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror 2024-09-14T09:11:20.514 DEBUG:tasks.rbd_mirror.cluster2.client.mirror.1:waiting for process to exit 2024-09-14T09:11:20.514 INFO:teuthology.orchestra.run:waiting for 300 2024-09-14T09:11:20.544 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi164.stderr:2024-09-14T09:11:20.544+0000 7fb097d32640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 (PID: 71165) UID: 1000 2024-09-14T09:11:20.545 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi164.stderr:2024-09-14T09:11:20.545+0000 7fb097531640 -1 rbd::mirror::InstanceWatcher: 0x5637d5226a80 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.6643 object: (2) No such file or directory 2024-09-14T09:11:20.545 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi164.stderr:2024-09-14T09:11:20.545+0000 7fb097531640 -1 rbd::mirror::InstanceWatcher: 0x5637d5226380 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.6643 object: (2) No such file or directory 2024-09-14T09:11:20.545 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi164.stderr:2024-09-14T09:11:20.545+0000 7fb082d08640 -1 rbd::mirror::InstanceWatcher: 0x5637d570c000 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2024-09-14T09:11:20.546 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi164.stderr:2024-09-14T09:11:20.545+0000 7faecbb1d640 -1 rbd::mirror::LeaderWatcher: 0x55b818586c00 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2024-09-14T09:11:20.547 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi164.stderr:2024-09-14T09:11:20.545+0000 7fb090d24640 -1 rbd::mirror::InstanceWatcher: 0x5637d5226a80 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2024-09-14T09:11:20.547 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi164.stderr:2024-09-14T09:11:20.545+0000 7fb090d24640 -1 rbd::mirror::InstanceWatcher: 0x5637d5226380 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2024-09-14T09:11:20.547 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi164.stderr:2024-09-14T09:11:20.545+0000 7fb089d16640 -1 rbd::mirror::MirrorStatusUpdater 0x5637d527e5a0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T09:11:20.547 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi164.stderr:2024-09-14T09:11:20.545+0000 7fb089d16640 -1 rbd::mirror::MirrorStatusUpdater 0x5637d527e360 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T09:11:20.547 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi164.stderr:2024-09-14T09:11:20.545+0000 7fb097531640 -1 rbd::mirror::NamespaceReplayer: 0x5637d527c000 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2024-09-14T09:11:20.547 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi164.stderr:2024-09-14T09:11:20.545+0000 7fb097531640 -1 rbd::mirror::NamespaceReplayer: 0x5637d527c1a0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2024-09-14T09:11:20.548 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi164.stderr:2024-09-14T09:11:20.546+0000 7faecc31e640 -1 rbd::mirror::InstanceWatcher: 0x55b81861e000 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.6029 object: (2) No such file or directory 2024-09-14T09:11:20.548 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi164.stderr:2024-09-14T09:11:20.546+0000 7fb090523640 -1 rbd::mirror::MirrorStatusUpdater 0x5637d527e000 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T09:11:20.549 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi164.stderr:2024-09-14T09:11:20.546+0000 7fb090523640 -1 rbd::mirror::MirrorStatusUpdater 0x5637d527e120 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T09:11:20.549 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi164.stderr:2024-09-14T09:11:20.546+0000 7fb096d30640 -1 rbd::mirror::NamespaceReplayer: 0x5637d527c000 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2024-09-14T09:11:20.549 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi164.stderr:2024-09-14T09:11:20.546+0000 7fb096d30640 -1 rbd::mirror::NamespaceReplayer: 0x5637d527c1a0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2024-09-14T09:11:20.550 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi164.stderr:2024-09-14T09:11:20.546+0000 7faec5b11640 -1 rbd::mirror::InstanceWatcher: 0x55b81861e000 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2024-09-14T09:11:20.550 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi164.stderr:2024-09-14T09:11:20.546+0000 7faebeb03640 -1 rbd::mirror::MirrorStatusUpdater 0x55b8183a27e0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T09:11:20.550 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi164.stderr:2024-09-14T09:11:20.546+0000 7faecc31e640 -1 rbd::mirror::NamespaceReplayer: 0x55b818290000 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2024-09-14T09:11:20.550 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi164.stderr:2024-09-14T09:11:20.546+0000 7faec5310640 -1 rbd::mirror::MirrorStatusUpdater 0x55b818371440 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T09:11:20.550 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi164.stderr:2024-09-14T09:11:20.546+0000 7faecbb1d640 -1 rbd::mirror::NamespaceReplayer: 0x55b818290000 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2024-09-14T09:11:20.552 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi164.stderr:2024-09-14T09:11:20.547+0000 7fb096d30640 -1 rbd::mirror::LeaderWatcher: 0x5637d5192c00 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2024-09-14T09:11:20.552 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi164.stderr:2024-09-14T09:11:20.548+0000 7fb096d30640 -1 rbd::mirror::InstanceWatcher: 0x5637d5226000 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.6643 object: (2) No such file or directory 2024-09-14T09:11:20.552 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi164.stderr:2024-09-14T09:11:20.548+0000 7fb090d24640 -1 rbd::mirror::InstanceWatcher: 0x5637d5226000 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2024-09-14T09:11:20.552 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi164.stderr:2024-09-14T09:11:20.548+0000 7fb08a517640 -1 rbd::mirror::MirrorStatusUpdater 0x5637d4ff25a0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T09:11:20.552 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi164.stderr:2024-09-14T09:11:20.548+0000 7fb096d30640 -1 rbd::mirror::NamespaceReplayer: 0x5637d4e2a1a0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2024-09-14T09:11:20.552 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi164.stderr:2024-09-14T09:11:20.548+0000 7fb090523640 -1 rbd::mirror::MirrorStatusUpdater 0x5637d4fb9560 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T09:11:20.552 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi164.stderr:2024-09-14T09:11:20.548+0000 7fb097531640 -1 rbd::mirror::NamespaceReplayer: 0x5637d4e2a1a0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2024-09-14T09:11:20.554 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi164.stderr:2024-09-14T09:11:20.549+0000 7faeb82f6640 -1 rbd::mirror::image_deleter::TrashWatcher: 0x55b8185fe240 handle_unregister_watcher: error unregistering watcher for trash directory: (2) No such file or directory 2024-09-14T09:11:20.555 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi164.stderr:2024-09-14T09:11:20.549+0000 7faeb7af5640 -1 rbd::mirror::PoolWatcher: 0x55b816841a40 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2024-09-14T09:11:20.555 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi164.stderr:2024-09-14T09:11:20.549+0000 7faeb1ae9640 -1 rbd::mirror::PoolWatcher: 0x55b817c325a0 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2024-09-14T09:11:20.555 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi164.stderr:2024-09-14T09:11:20.550+0000 7faecbb1d640 -1 rbd::mirror::LeaderWatcher: 0x55b817961200 handle_notify_lock_released: error notifying leader lock released: (2) No such file or directory 2024-09-14T09:11:20.555 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi164.stderr:2024-09-14T09:11:20.550+0000 7faecbb1d640 -1 rbd::mirror::LeaderWatcher: 0x55b817961200 handle_shut_down_leader_lock: error shutting down leader lock: (2) No such file or directory 2024-09-14T09:11:20.555 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi164.stderr:2024-09-14T09:11:20.550+0000 7faecc31e640 -1 rbd::mirror::LeaderWatcher: 0x55b817961200 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2024-09-14T09:11:20.555 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi164.stderr:2024-09-14T09:11:20.550+0000 7faecc31e640 -1 rbd::mirror::InstanceWatcher: 0x55b817c00000 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.6031 object: (2) No such file or directory 2024-09-14T09:11:20.555 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi164.stderr:2024-09-14T09:11:20.551+0000 7faeb82f6640 -1 rbd::mirror::InstanceWatcher: 0x55b817c00000 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2024-09-14T09:11:20.555 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi164.stderr:2024-09-14T09:11:20.551+0000 7faeb12e8640 -1 rbd::mirror::MirrorStatusUpdater 0x55b8177ae6c0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T09:11:20.556 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi164.stderr:2024-09-14T09:11:20.551+0000 7faecbb1d640 -1 rbd::mirror::NamespaceReplayer: 0x55b81779c4e0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2024-09-14T09:11:20.556 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi164.stderr:2024-09-14T09:11:20.551+0000 7faeb7af5640 -1 rbd::mirror::MirrorStatusUpdater 0x55b817963d40 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T09:11:20.556 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi164.stderr:2024-09-14T09:11:20.551+0000 7faecc31e640 -1 rbd::mirror::NamespaceReplayer: 0x55b81779c4e0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2024-09-14T09:11:20.561 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Stopped 2024-09-14T09:11:20.562 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror 2024-09-14T09:11:20.572 DEBUG:tasks.rbd_mirror.cluster2.client.mirror.0:waiting for process to exit 2024-09-14T09:11:20.572 INFO:teuthology.orchestra.run:waiting for 300 2024-09-14T09:11:20.603 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi164.stderr:2024-09-14T09:11:20.603+0000 7fe452f38640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 (PID: 61521) UID: 1000 2024-09-14T09:11:20.604 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi164.stderr:2024-09-14T09:11:20.604+0000 7fe451f36640 -1 rbd::mirror::LeaderWatcher: 0x55632bbc9200 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2024-09-14T09:11:20.606 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi164.stderr:2024-09-14T09:11:20.604+0000 7fe452737640 -1 rbd::mirror::InstanceWatcher: 0x55632b9aa380 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5929 object: (2) No such file or directory 2024-09-14T09:11:20.606 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi164.stderr:2024-09-14T09:11:20.604+0000 7fe452737640 -1 rbd::mirror::InstanceWatcher: 0x55632b9aa700 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5929 object: (2) No such file or directory 2024-09-14T09:11:20.606 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi164.stderr:2024-09-14T09:11:20.604+0000 7fe452737640 -1 rbd::mirror::InstanceWatcher: 0x55632be9e000 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5931 object: (2) No such file or directory 2024-09-14T09:11:20.606 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi164.stderr:2024-09-14T09:11:20.604+0000 7fe44b729640 -1 rbd::mirror::InstanceWatcher: 0x55632b9aa380 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2024-09-14T09:11:20.606 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi164.stderr:2024-09-14T09:11:20.604+0000 7fe44bf2a640 -1 rbd::mirror::InstanceWatcher: 0x55632b9aa700 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2024-09-14T09:11:20.606 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi164.stderr:2024-09-14T09:11:20.604+0000 7fe43e70f640 -1 rbd::mirror::InstanceWatcher: 0x55632be9e000 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2024-09-14T09:11:20.606 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi164.stderr:2024-09-14T09:11:20.605+0000 7fe44571d640 -1 rbd::mirror::MirrorStatusUpdater 0x55632ba025a0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T09:11:20.606 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi164.stderr:2024-09-14T09:11:20.605+0000 7fe44571d640 -1 rbd::mirror::MirrorStatusUpdater 0x55632ba02360 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T09:11:20.606 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi164.stderr:2024-09-14T09:11:20.605+0000 7fe452737640 -1 rbd::mirror::NamespaceReplayer: 0x55632ba00000 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2024-09-14T09:11:20.607 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi164.stderr:2024-09-14T09:11:20.605+0000 7fe452737640 -1 rbd::mirror::NamespaceReplayer: 0x55632ba001a0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2024-09-14T09:11:20.607 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi164.stderr:2024-09-14T09:11:20.605+0000 7fe437701640 -1 rbd::mirror::MirrorStatusUpdater 0x55632bc46900 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T09:11:20.607 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi164.stderr:2024-09-14T09:11:20.605+0000 7fe451f36640 -1 rbd::mirror::NamespaceReplayer: 0x55632bea0000 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2024-09-14T09:11:20.607 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi164.stderr:2024-09-14T09:11:20.605+0000 7fe44bf2a640 -1 rbd::mirror::MirrorStatusUpdater 0x55632ba02000 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T09:11:20.607 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi164.stderr:2024-09-14T09:11:20.605+0000 7fe44bf2a640 -1 rbd::mirror::MirrorStatusUpdater 0x55632ba02120 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T09:11:20.607 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi164.stderr:2024-09-14T09:11:20.605+0000 7fe452737640 -1 rbd::mirror::NamespaceReplayer: 0x55632ba00000 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2024-09-14T09:11:20.607 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi164.stderr:2024-09-14T09:11:20.605+0000 7fe452737640 -1 rbd::mirror::NamespaceReplayer: 0x55632ba001a0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2024-09-14T09:11:20.607 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi164.stderr:2024-09-14T09:11:20.605+0000 7fe43df0e640 -1 rbd::mirror::MirrorStatusUpdater 0x55632bc319e0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T09:11:20.607 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi164.stderr:2024-09-14T09:11:20.605+0000 7fe451f36640 -1 rbd::mirror::NamespaceReplayer: 0x55632bea0000 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2024-09-14T09:11:20.608 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi164.stderr:2024-09-14T09:11:20.607+0000 7fe451f36640 -1 rbd::mirror::LeaderWatcher: 0x55632b902c00 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2024-09-14T09:11:20.608 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi164.stderr:2024-09-14T09:11:20.607+0000 7fe451f36640 -1 rbd::mirror::InstanceWatcher: 0x55632b9aa000 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5929 object: (2) No such file or directory 2024-09-14T09:11:20.608 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi164.stderr:2024-09-14T09:11:20.608+0000 7fe44b729640 -1 rbd::mirror::InstanceWatcher: 0x55632b9aa000 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2024-09-14T09:11:20.609 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi164.stderr:2024-09-14T09:11:20.608+0000 7fe444f1c640 -1 rbd::mirror::MirrorStatusUpdater 0x55632b76e6c0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T09:11:20.609 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi164.stderr:2024-09-14T09:11:20.608+0000 7fe452737640 -1 rbd::mirror::NamespaceReplayer: 0x55632b5de1a0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2024-09-14T09:11:20.609 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi164.stderr:2024-09-14T09:11:20.608+0000 7fe44bf2a640 -1 rbd::mirror::MirrorStatusUpdater 0x55632b717320 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T09:11:20.609 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi164.stderr:2024-09-14T09:11:20.608+0000 7fe451f36640 -1 rbd::mirror::NamespaceReplayer: 0x55632b5de1a0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2024-09-14T09:11:20.620 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Stopped 2024-09-14T09:11:20.621 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror 2024-09-14T09:11:20.633 DEBUG:tasks.rbd_mirror.cluster1.client.mirror.3:waiting for process to exit 2024-09-14T09:11:20.633 INFO:teuthology.orchestra.run:waiting for 300 2024-09-14T09:11:20.662 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi164.stderr:2024-09-14T09:11:20.662+0000 7fbeb5c5c640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.3 (PID: 73579) UID: 1000 2024-09-14T09:11:20.672 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Stopped 2024-09-14T09:11:20.672 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror 2024-09-14T09:11:20.682 DEBUG:tasks.rbd_mirror.cluster1.client.mirror.2:waiting for process to exit 2024-09-14T09:11:20.682 INFO:teuthology.orchestra.run:waiting for 300 2024-09-14T09:11:20.713 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi164.stderr:2024-09-14T09:11:20.713+0000 7fc62769d640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 (PID: 72184) UID: 1000 2024-09-14T09:11:20.715 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi164.stderr:2024-09-14T09:11:20.714+0000 7fc626e9c640 -1 rbd::mirror::LeaderWatcher: 0x563d4f77d200 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2024-09-14T09:11:20.715 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi164.stderr:2024-09-14T09:11:20.714+0000 7fc626e9c640 -1 rbd::mirror::InstanceWatcher: 0x563d4f54a380 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.6149 object: (2) No such file or directory 2024-09-14T09:11:20.715 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi164.stderr:2024-09-14T09:11:20.714+0000 7fc626e9c640 -1 rbd::mirror::InstanceWatcher: 0x563d4f54aa80 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.6149 object: (2) No such file or directory 2024-09-14T09:11:20.715 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi164.stderr:2024-09-14T09:11:20.714+0000 7fc626e9c640 -1 rbd::mirror::InstanceWatcher: 0x563d4f54a700 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.6152 object: (2) No such file or directory 2024-09-14T09:11:20.715 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi164.stderr:2024-09-14T09:11:20.714+0000 7fc62068f640 -1 rbd::mirror::InstanceWatcher: 0x563d4f54a380 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2024-09-14T09:11:20.715 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi164.stderr:2024-09-14T09:11:20.714+0000 7fc62068f640 -1 rbd::mirror::InstanceWatcher: 0x563d4f54aa80 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2024-09-14T09:11:20.715 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi164.stderr:2024-09-14T09:11:20.714+0000 7fc612673640 -1 rbd::mirror::InstanceWatcher: 0x563d4f54a700 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2024-09-14T09:11:20.716 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi164.stderr:2024-09-14T09:11:20.714+0000 7fc619681640 -1 rbd::mirror::MirrorStatusUpdater 0x563d4f59e5a0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T09:11:20.716 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi164.stderr:2024-09-14T09:11:20.714+0000 7fc619681640 -1 rbd::mirror::MirrorStatusUpdater 0x563d4f59e360 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T09:11:20.716 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi164.stderr:2024-09-14T09:11:20.714+0000 7fc626e9c640 -1 rbd::mirror::NamespaceReplayer: 0x563d4f59c000 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2024-09-14T09:11:20.716 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi164.stderr:2024-09-14T09:11:20.714+0000 7fc626e9c640 -1 rbd::mirror::NamespaceReplayer: 0x563d4f59c1a0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2024-09-14T09:11:20.716 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi164.stderr:2024-09-14T09:11:20.715+0000 7fc61fe8e640 -1 rbd::mirror::MirrorStatusUpdater 0x563d4f59e000 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T09:11:20.716 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi164.stderr:2024-09-14T09:11:20.715+0000 7fc61fe8e640 -1 rbd::mirror::MirrorStatusUpdater 0x563d4f59e120 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T09:11:20.716 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi164.stderr:2024-09-14T09:11:20.715+0000 7fc62669b640 -1 rbd::mirror::NamespaceReplayer: 0x563d4f59c000 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2024-09-14T09:11:20.716 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi164.stderr:2024-09-14T09:11:20.715+0000 7fc60c667640 -1 rbd::mirror::MirrorStatusUpdater 0x563d4f7b27e0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T09:11:20.716 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi164.stderr:2024-09-14T09:11:20.715+0000 7fc62669b640 -1 rbd::mirror::NamespaceReplayer: 0x563d4f59c1a0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2024-09-14T09:11:20.716 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi164.stderr:2024-09-14T09:11:20.715+0000 7fc62669b640 -1 rbd::mirror::NamespaceReplayer: 0x563d4fa16000 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2024-09-14T09:11:20.716 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi164.stderr:2024-09-14T09:11:20.715+0000 7fc612e74640 -1 rbd::mirror::MirrorStatusUpdater 0x563d4f7818c0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T09:11:20.716 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi164.stderr:2024-09-14T09:11:20.715+0000 7fc626e9c640 -1 rbd::mirror::NamespaceReplayer: 0x563d4fa16000 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2024-09-14T09:11:20.717 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi164.stderr:2024-09-14T09:11:20.717+0000 7fc62669b640 -1 rbd::mirror::LeaderWatcher: 0x563d4f4aac00 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2024-09-14T09:11:20.717 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi164.stderr:2024-09-14T09:11:20.717+0000 7fc62669b640 -1 rbd::mirror::InstanceWatcher: 0x563d4f54a000 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.6149 object: (2) No such file or directory 2024-09-14T09:11:20.718 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi164.stderr:2024-09-14T09:11:20.718+0000 7fc62068f640 -1 rbd::mirror::InstanceWatcher: 0x563d4f54a000 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2024-09-14T09:11:20.718 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi164.stderr:2024-09-14T09:11:20.718+0000 7fc619e82640 -1 rbd::mirror::MirrorStatusUpdater 0x563d4f3086c0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T09:11:20.718 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi164.stderr:2024-09-14T09:11:20.718+0000 7fc62669b640 -1 rbd::mirror::NamespaceReplayer: 0x563d4e387520 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2024-09-14T09:11:20.718 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi164.stderr:2024-09-14T09:11:20.718+0000 7fc61fe8e640 -1 rbd::mirror::MirrorStatusUpdater 0x563d4f299320 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T09:11:20.719 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi164.stderr:2024-09-14T09:11:20.718+0000 7fc626e9c640 -1 rbd::mirror::NamespaceReplayer: 0x563d4e387520 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2024-09-14T09:11:20.729 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Stopped 2024-09-14T09:11:20.729 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror 2024-09-14T09:11:20.740 DEBUG:tasks.rbd_mirror.cluster1.client.mirror.1:waiting for process to exit 2024-09-14T09:11:20.740 INFO:teuthology.orchestra.run:waiting for 300 2024-09-14T09:11:20.770 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi164.stderr:2024-09-14T09:11:20.770+0000 7f0c0c2b4640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 (PID: 59203) UID: 1000 2024-09-14T09:11:20.770 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi164.stderr:2024-09-14T09:11:20.770+0000 7f0c0bab3640 -1 rbd::mirror::LeaderWatcher: 0x560c3a9b9200 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2024-09-14T09:11:20.771 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi164.stderr:2024-09-14T09:11:20.771+0000 7f0c0b2b2640 -1 rbd::mirror::InstanceWatcher: 0x560c3a7a0a80 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5435 object: (2) No such file or directory 2024-09-14T09:11:20.771 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi164.stderr:2024-09-14T09:11:20.771+0000 7f0bf7a8b640 -1 rbd::mirror::InstanceWatcher: 0x560c3a7a0a80 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2024-09-14T09:11:20.772 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi164.stderr:2024-09-14T09:11:20.772+0000 7f0bf0a7d640 -1 rbd::mirror::MirrorStatusUpdater 0x560c3a7f86c0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T09:11:20.772 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi164.stderr:2024-09-14T09:11:20.772+0000 7f0c0bab3640 -1 rbd::mirror::NamespaceReplayer: 0x560c3a7f64e0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2024-09-14T09:11:20.772 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi164.stderr:2024-09-14T09:11:20.772+0000 7f0bf728a640 -1 rbd::mirror::MirrorStatusUpdater 0x560c3a9ddd40 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T09:11:20.772 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi164.stderr:2024-09-14T09:11:20.772+0000 7f0c0b2b2640 -1 rbd::mirror::NamespaceReplayer: 0x560c3a7f64e0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2024-09-14T09:11:20.784 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Stopped 2024-09-14T09:11:20.784 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror 2024-09-14T09:11:20.795 DEBUG:tasks.rbd_mirror.cluster1.client.mirror.0:waiting for process to exit 2024-09-14T09:11:20.795 INFO:teuthology.orchestra.run:waiting for 300 2024-09-14T09:11:20.825 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi164.stderr:2024-09-14T09:11:20.825+0000 7faeccb1f640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 (PID: 68779) UID: 1000 2024-09-14T09:11:20.834 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Stopped 2024-09-14T09:11:20.834 DEBUG:teuthology.run_tasks:Unwinding manager ceph 2024-09-14T09:11:20.845 INFO:tasks.ceph.ceph_manager.cluster2:waiting for clean 2024-09-14T09:11:20.845 DEBUG:teuthology.orchestra.run.smithi164:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2024-09-14T09:11:21.155 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T09:11:21.156 INFO:teuthology.orchestra.run.smithi164.stderr:dumped all 2024-09-14T09:11:21.169 INFO:teuthology.orchestra.run.smithi164.stdout:{"pg_ready":true,"pg_map":{"version":1522,"stamp":"2024-09-14T09:11:19.688759+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":81,"num_write_kb":1108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":274,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":2828580,"kb_used_data":60276,"kb_used_omap":4,"kb_used_meta":2768251,"kb_avail":280286940,"statfs":{"total":289910292480,"available":287013826560,"internally_reserved":0,"allocated":61722624,"data_stored":656342099,"data_compressed":977025,"data_compressed_allocated":42258432,"data_compressed_original":643944448,"omap_allocated":4785,"internal_metadata":2834689359},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-536870966,"num_objects":-164,"num_object_clones":0,"num_object_copies":-328,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-164,"num_whiteouts":0,"num_read":-115372,"num_read_kb":-39629426,"num_write":-406689,"num_write_kb":-148835171,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-7,"num_bytes_recovered":-12582931,"num_keys_recovered":0,"num_objects_omap":-14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.309703"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":777,"reported_epoch":386,"state":"active+clean","last_fresh":"2024-09-14T09:10:45.806042+0000","last_change":"2024-09-14T08:33:33.870517+0000","last_active":"2024-09-14T09:10:45.806042+0000","last_peered":"2024-09-14T09:10:45.806042+0000","last_clean":"2024-09-14T09:10:45.806042+0000","last_became_active":"2024-09-14T08:33:31.868938+0000","last_became_peered":"2024-09-14T08:33:31.868938+0000","last_unstale":"2024-09-14T09:10:45.806042+0000","last_undegraded":"2024-09-14T09:10:45.806042+0000","last_fullsized":"2024-09-14T09:10:45.806042+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:33:30.860386+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:33:30.860386+0000","last_clean_scrub_stamp":"2024-09-14T08:33:30.860386+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:25:40.377777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00071425700000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":777,"reported_epoch":386,"state":"active+clean","last_fresh":"2024-09-14T09:10:45.805797+0000","last_change":"2024-09-14T08:33:33.870929+0000","last_active":"2024-09-14T09:10:45.805797+0000","last_peered":"2024-09-14T09:10:45.805797+0000","last_clean":"2024-09-14T09:10:45.805797+0000","last_became_active":"2024-09-14T08:33:31.868362+0000","last_became_peered":"2024-09-14T08:33:31.868362+0000","last_unstale":"2024-09-14T09:10:45.805797+0000","last_undegraded":"2024-09-14T09:10:45.805797+0000","last_fullsized":"2024-09-14T09:10:45.805797+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:33:30.860386+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:33:30.860386+0000","last_clean_scrub_stamp":"2024-09-14T08:33:30.860386+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:59:01.028926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001256898,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":778,"reported_epoch":386,"state":"active+clean","last_fresh":"2024-09-14T09:10:45.806062+0000","last_change":"2024-09-14T08:33:33.870538+0000","last_active":"2024-09-14T09:10:45.806062+0000","last_peered":"2024-09-14T09:10:45.806062+0000","last_clean":"2024-09-14T09:10:45.806062+0000","last_became_active":"2024-09-14T08:33:31.868180+0000","last_became_peered":"2024-09-14T08:33:31.868180+0000","last_unstale":"2024-09-14T09:10:45.806062+0000","last_undegraded":"2024-09-14T09:10:45.806062+0000","last_fullsized":"2024-09-14T09:10:45.806062+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:33:30.860386+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:33:30.860386+0000","last_clean_scrub_stamp":"2024-09-14T08:33:30.860386+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T19:51:08.921688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00044658000000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":778,"reported_epoch":386,"state":"active+clean","last_fresh":"2024-09-14T09:10:45.804505+0000","last_change":"2024-09-14T08:33:33.870794+0000","last_active":"2024-09-14T09:10:45.804505+0000","last_peered":"2024-09-14T09:10:45.804505+0000","last_clean":"2024-09-14T09:10:45.804505+0000","last_became_active":"2024-09-14T08:33:31.868126+0000","last_became_peered":"2024-09-14T08:33:31.868126+0000","last_unstale":"2024-09-14T09:10:45.804505+0000","last_undegraded":"2024-09-14T09:10:45.804505+0000","last_fullsized":"2024-09-14T09:10:45.804505+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:33:30.860386+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:33:30.860386+0000","last_clean_scrub_stamp":"2024-09-14T08:33:30.860386+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:20:30.169758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064856099999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"16'2","reported_seq":782,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-09-14T09:11:18.873965+0000","last_change":"2024-09-14T08:33:33.865752+0000","last_active":"2024-09-14T09:11:18.873965+0000","last_peered":"2024-09-14T09:11:18.873965+0000","last_clean":"2024-09-14T09:11:18.873965+0000","last_became_active":"2024-09-14T08:33:31.870304+0000","last_became_peered":"2024-09-14T08:33:31.870304+0000","last_unstale":"2024-09-14T09:11:18.873965+0000","last_undegraded":"2024-09-14T09:11:18.873965+0000","last_fullsized":"2024-09-14T09:11:18.873965+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:33:30.860386+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:33:30.860386+0000","last_clean_scrub_stamp":"2024-09-14T08:33:30.860386+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:17:46.077907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000475141,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":778,"reported_epoch":386,"state":"active+clean","last_fresh":"2024-09-14T09:10:45.650939+0000","last_change":"2024-09-14T08:33:34.137500+0000","last_active":"2024-09-14T09:10:45.650939+0000","last_peered":"2024-09-14T09:10:45.650939+0000","last_clean":"2024-09-14T09:10:45.650939+0000","last_became_active":"2024-09-14T08:33:31.869506+0000","last_became_peered":"2024-09-14T08:33:31.869506+0000","last_unstale":"2024-09-14T09:10:45.650939+0000","last_undegraded":"2024-09-14T09:10:45.650939+0000","last_fullsized":"2024-09-14T09:10:45.650939+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:33:30.860386+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:33:30.860386+0000","last_clean_scrub_stamp":"2024-09-14T08:33:30.860386+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:37:11.985802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036617900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":778,"reported_epoch":386,"state":"active+clean","last_fresh":"2024-09-14T09:10:45.650739+0000","last_change":"2024-09-14T08:33:34.137555+0000","last_active":"2024-09-14T09:10:45.650739+0000","last_peered":"2024-09-14T09:10:45.650739+0000","last_clean":"2024-09-14T09:10:45.650739+0000","last_became_active":"2024-09-14T08:33:31.868768+0000","last_became_peered":"2024-09-14T08:33:31.868768+0000","last_unstale":"2024-09-14T09:10:45.650739+0000","last_undegraded":"2024-09-14T09:10:45.650739+0000","last_fullsized":"2024-09-14T09:10:45.650739+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:33:30.860386+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:33:30.860386+0000","last_clean_scrub_stamp":"2024-09-14T08:33:30.860386+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:16:44.486807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041116299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"14'1","reported_seq":779,"reported_epoch":386,"state":"active+clean","last_fresh":"2024-09-14T09:10:45.805228+0000","last_change":"2024-09-14T08:33:33.870202+0000","last_active":"2024-09-14T09:10:45.805228+0000","last_peered":"2024-09-14T09:10:45.805228+0000","last_clean":"2024-09-14T09:10:45.805228+0000","last_became_active":"2024-09-14T08:33:31.868308+0000","last_became_peered":"2024-09-14T08:33:31.868308+0000","last_unstale":"2024-09-14T09:10:45.805228+0000","last_undegraded":"2024-09-14T09:10:45.805228+0000","last_fullsized":"2024-09-14T09:10:45.805228+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:33:30.860386+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:33:30.860386+0000","last_clean_scrub_stamp":"2024-09-14T08:33:30.860386+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T17:51:35.908288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00058585600000000005,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"12'52","reported_seq":844,"reported_epoch":386,"state":"active+clean","last_fresh":"2024-09-14T09:10:45.804354+0000","last_change":"2024-09-14T08:33:29.863094+0000","last_active":"2024-09-14T09:10:45.804354+0000","last_peered":"2024-09-14T09:10:45.804354+0000","last_clean":"2024-09-14T09:10:45.804354+0000","last_became_active":"2024-09-14T08:33:29.862802+0000","last_became_peered":"2024-09-14T08:33:29.862802+0000","last_unstale":"2024-09-14T09:10:45.804354+0000","last_undegraded":"2024-09-14T09:10:45.804354+0000","last_fullsized":"2024-09-14T09:10:45.804354+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:33:28.845692+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:33:28.845692+0000","last_clean_scrub_stamp":"2024-09-14T08:33:28.845692+0000","objects_scrubbed":0,"log_size":52,"log_dups_size":0,"ondisk_log_size":52,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:35:27.535389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":79,"num_write_kb":1106,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":79,"num_write_kb":1106,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":1180736,"data_compressed":7088,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":52,"ondisk_log_size":52,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654706124,"num_pgs":80,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":835848,"kb_used_data":27640,"kb_used_omap":1,"kb_used_meta":808190,"kb_avail":93535992,"statfs":{"total":96636764160,"available":95780855808,"internally_reserved":0,"allocated":28303360,"data_stored":309955601,"data_compressed":460454,"data_compressed_allocated":20037632,"data_compressed_original":304523264,"omap_allocated":1595,"internal_metadata":827587013},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":9,"seq":38654706126,"num_pgs":98,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":950640,"kb_used_data":29472,"kb_used_omap":1,"kb_used_meta":921150,"kb_avail":93421200,"statfs":{"total":96636764160,"available":95663308800,"internally_reserved":0,"allocated":30179328,"data_stored":344100366,"data_compressed":513027,"data_compressed_allocated":22183936,"data_compressed_original":338847744,"omap_allocated":1595,"internal_metadata":943258053},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":9,"seq":38654706125,"num_pgs":96,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1042092,"kb_used_data":3164,"kb_used_omap":1,"kb_used_meta":1038910,"kb_avail":93329748,"statfs":{"total":96636764160,"available":95569661952,"internally_reserved":0,"allocated":3239936,"data_stored":2286132,"data_compressed":3544,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1595,"internal_metadata":1063844293},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":3544,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":3544,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T09:11:21.170 DEBUG:teuthology.orchestra.run.smithi164:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2024-09-14T09:11:21.454 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T09:11:21.455 INFO:teuthology.orchestra.run.smithi164.stderr:dumped all 2024-09-14T09:11:21.467 INFO:teuthology.orchestra.run.smithi164.stdout:{"pg_ready":true,"pg_map":{"version":1523,"stamp":"2024-09-14T09:11:21.381532+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":81,"num_write_kb":1108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":274,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":2818012,"kb_used_data":9444,"kb_used_omap":4,"kb_used_meta":2808507,"kb_avail":280297508,"statfs":{"total":289910292480,"available":287024648192,"internally_reserved":0,"allocated":9670656,"data_stored":6276160,"data_compressed":7088,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":4785,"internal_metadata":2875911503},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":15,"apply_latency_ms":15,"commit_latency_ns":15000000,"apply_latency_ns":15000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-536870966,"num_objects":-164,"num_object_clones":0,"num_object_copies":-328,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-164,"num_whiteouts":0,"num_read":-115372,"num_read_kb":-39629426,"num_write":-406689,"num_write_kb":-148835171,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-7,"num_bytes_recovered":-12582931,"num_keys_recovered":0,"num_objects_omap":-14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.001930"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":779,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-09-14T09:11:17.691832+0000","last_change":"2024-09-14T08:33:33.870517+0000","last_active":"2024-09-14T09:11:17.691832+0000","last_peered":"2024-09-14T09:11:17.691832+0000","last_clean":"2024-09-14T09:11:17.691832+0000","last_became_active":"2024-09-14T08:33:31.868938+0000","last_became_peered":"2024-09-14T08:33:31.868938+0000","last_unstale":"2024-09-14T09:11:17.691832+0000","last_undegraded":"2024-09-14T09:11:17.691832+0000","last_fullsized":"2024-09-14T09:11:17.691832+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:33:30.860386+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:33:30.860386+0000","last_clean_scrub_stamp":"2024-09-14T08:33:30.860386+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:25:40.377777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00071425700000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":779,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-09-14T09:11:17.695468+0000","last_change":"2024-09-14T08:33:33.870929+0000","last_active":"2024-09-14T09:11:17.695468+0000","last_peered":"2024-09-14T09:11:17.695468+0000","last_clean":"2024-09-14T09:11:17.695468+0000","last_became_active":"2024-09-14T08:33:31.868362+0000","last_became_peered":"2024-09-14T08:33:31.868362+0000","last_unstale":"2024-09-14T09:11:17.695468+0000","last_undegraded":"2024-09-14T09:11:17.695468+0000","last_fullsized":"2024-09-14T09:11:17.695468+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:33:30.860386+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:33:30.860386+0000","last_clean_scrub_stamp":"2024-09-14T08:33:30.860386+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:59:01.028926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001256898,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":780,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-09-14T09:11:17.696979+0000","last_change":"2024-09-14T08:33:33.870538+0000","last_active":"2024-09-14T09:11:17.696979+0000","last_peered":"2024-09-14T09:11:17.696979+0000","last_clean":"2024-09-14T09:11:17.696979+0000","last_became_active":"2024-09-14T08:33:31.868180+0000","last_became_peered":"2024-09-14T08:33:31.868180+0000","last_unstale":"2024-09-14T09:11:17.696979+0000","last_undegraded":"2024-09-14T09:11:17.696979+0000","last_fullsized":"2024-09-14T09:11:17.696979+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:33:30.860386+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:33:30.860386+0000","last_clean_scrub_stamp":"2024-09-14T08:33:30.860386+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T19:51:08.921688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00044658000000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":780,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-09-14T09:11:17.690270+0000","last_change":"2024-09-14T08:33:33.870794+0000","last_active":"2024-09-14T09:11:17.690270+0000","last_peered":"2024-09-14T09:11:17.690270+0000","last_clean":"2024-09-14T09:11:17.690270+0000","last_became_active":"2024-09-14T08:33:31.868126+0000","last_became_peered":"2024-09-14T08:33:31.868126+0000","last_unstale":"2024-09-14T09:11:17.690270+0000","last_undegraded":"2024-09-14T09:11:17.690270+0000","last_fullsized":"2024-09-14T09:11:17.690270+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:33:30.860386+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:33:30.860386+0000","last_clean_scrub_stamp":"2024-09-14T08:33:30.860386+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:20:30.169758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064856099999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"16'2","reported_seq":782,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-09-14T09:11:18.873965+0000","last_change":"2024-09-14T08:33:33.865752+0000","last_active":"2024-09-14T09:11:18.873965+0000","last_peered":"2024-09-14T09:11:18.873965+0000","last_clean":"2024-09-14T09:11:18.873965+0000","last_became_active":"2024-09-14T08:33:31.870304+0000","last_became_peered":"2024-09-14T08:33:31.870304+0000","last_unstale":"2024-09-14T09:11:18.873965+0000","last_undegraded":"2024-09-14T09:11:18.873965+0000","last_fullsized":"2024-09-14T09:11:18.873965+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:33:30.860386+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:33:30.860386+0000","last_clean_scrub_stamp":"2024-09-14T08:33:30.860386+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:17:46.077907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000475141,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":780,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-09-14T09:11:18.026436+0000","last_change":"2024-09-14T08:33:34.137500+0000","last_active":"2024-09-14T09:11:18.026436+0000","last_peered":"2024-09-14T09:11:18.026436+0000","last_clean":"2024-09-14T09:11:18.026436+0000","last_became_active":"2024-09-14T08:33:31.869506+0000","last_became_peered":"2024-09-14T08:33:31.869506+0000","last_unstale":"2024-09-14T09:11:18.026436+0000","last_undegraded":"2024-09-14T09:11:18.026436+0000","last_fullsized":"2024-09-14T09:11:18.026436+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:33:30.860386+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:33:30.860386+0000","last_clean_scrub_stamp":"2024-09-14T08:33:30.860386+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:37:11.985802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036617900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":780,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-09-14T09:11:18.027672+0000","last_change":"2024-09-14T08:33:34.137555+0000","last_active":"2024-09-14T09:11:18.027672+0000","last_peered":"2024-09-14T09:11:18.027672+0000","last_clean":"2024-09-14T09:11:18.027672+0000","last_became_active":"2024-09-14T08:33:31.868768+0000","last_became_peered":"2024-09-14T08:33:31.868768+0000","last_unstale":"2024-09-14T09:11:18.027672+0000","last_undegraded":"2024-09-14T09:11:18.027672+0000","last_fullsized":"2024-09-14T09:11:18.027672+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:33:30.860386+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:33:30.860386+0000","last_clean_scrub_stamp":"2024-09-14T08:33:30.860386+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:16:44.486807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041116299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"14'1","reported_seq":781,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-09-14T09:11:17.693595+0000","last_change":"2024-09-14T08:33:33.870202+0000","last_active":"2024-09-14T09:11:17.693595+0000","last_peered":"2024-09-14T09:11:17.693595+0000","last_clean":"2024-09-14T09:11:17.693595+0000","last_became_active":"2024-09-14T08:33:31.868308+0000","last_became_peered":"2024-09-14T08:33:31.868308+0000","last_unstale":"2024-09-14T09:11:17.693595+0000","last_undegraded":"2024-09-14T09:11:17.693595+0000","last_fullsized":"2024-09-14T09:11:17.693595+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:33:30.860386+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:33:30.860386+0000","last_clean_scrub_stamp":"2024-09-14T08:33:30.860386+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T17:51:35.908288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00058585600000000005,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"12'52","reported_seq":846,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-09-14T09:11:17.691404+0000","last_change":"2024-09-14T08:33:29.863094+0000","last_active":"2024-09-14T09:11:17.691404+0000","last_peered":"2024-09-14T09:11:17.691404+0000","last_clean":"2024-09-14T09:11:17.691404+0000","last_became_active":"2024-09-14T08:33:29.862802+0000","last_became_peered":"2024-09-14T08:33:29.862802+0000","last_unstale":"2024-09-14T09:11:17.691404+0000","last_undegraded":"2024-09-14T09:11:17.691404+0000","last_fullsized":"2024-09-14T09:11:17.691404+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:33:28.845692+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:33:28.845692+0000","last_clean_scrub_stamp":"2024-09-14T08:33:28.845692+0000","objects_scrubbed":0,"log_size":52,"log_dups_size":0,"ondisk_log_size":52,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:35:27.535389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":79,"num_write_kb":1106,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":79,"num_write_kb":1106,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":1180736,"data_compressed":7088,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":52,"ondisk_log_size":52,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654706125,"num_pgs":80,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":829372,"kb_used_data":3112,"kb_used_omap":1,"kb_used_meta":826238,"kb_avail":93542468,"statfs":{"total":96636764160,"available":95787487232,"internally_reserved":0,"allocated":3186688,"data_stored":1699830,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":846068165},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[]},{"osd":1,"up_from":9,"seq":38654706127,"num_pgs":98,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":946548,"kb_used_data":3168,"kb_used_omap":1,"kb_used_meta":943358,"kb_avail":93425292,"statfs":{"total":96636764160,"available":95667499008,"internally_reserved":0,"allocated":3244032,"data_stored":2290198,"data_compressed":3544,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1595,"internal_metadata":965999045},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[]},{"osd":0,"up_from":9,"seq":38654706125,"num_pgs":96,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1042092,"kb_used_data":3164,"kb_used_omap":1,"kb_used_meta":1038910,"kb_avail":93329748,"statfs":{"total":96636764160,"available":95569661952,"internally_reserved":0,"allocated":3239936,"data_stored":2286132,"data_compressed":3544,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1595,"internal_metadata":1063844293},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":3544,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":3544,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T09:11:21.468 INFO:tasks.ceph.ceph_manager.cluster2:clean! 2024-09-14T09:11:21.468 DEBUG:teuthology.orchestra.run.smithi164:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2024-09-14T09:11:21.749 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T09:11:21.749 INFO:teuthology.orchestra.run.smithi164.stderr:dumped all 2024-09-14T09:11:21.761 INFO:teuthology.orchestra.run.smithi164.stdout:{"pg_ready":true,"pg_map":{"version":1523,"stamp":"2024-09-14T09:11:21.381532+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":81,"num_write_kb":1108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":274,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":2818012,"kb_used_data":9444,"kb_used_omap":4,"kb_used_meta":2808507,"kb_avail":280297508,"statfs":{"total":289910292480,"available":287024648192,"internally_reserved":0,"allocated":9670656,"data_stored":6276160,"data_compressed":7088,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":4785,"internal_metadata":2875911503},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":15,"apply_latency_ms":15,"commit_latency_ns":15000000,"apply_latency_ns":15000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-536870966,"num_objects":-164,"num_object_clones":0,"num_object_copies":-328,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-164,"num_whiteouts":0,"num_read":-115372,"num_read_kb":-39629426,"num_write":-406689,"num_write_kb":-148835171,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-7,"num_bytes_recovered":-12582931,"num_keys_recovered":0,"num_objects_omap":-14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.001930"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":779,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-09-14T09:11:17.691832+0000","last_change":"2024-09-14T08:33:33.870517+0000","last_active":"2024-09-14T09:11:17.691832+0000","last_peered":"2024-09-14T09:11:17.691832+0000","last_clean":"2024-09-14T09:11:17.691832+0000","last_became_active":"2024-09-14T08:33:31.868938+0000","last_became_peered":"2024-09-14T08:33:31.868938+0000","last_unstale":"2024-09-14T09:11:17.691832+0000","last_undegraded":"2024-09-14T09:11:17.691832+0000","last_fullsized":"2024-09-14T09:11:17.691832+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:33:30.860386+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:33:30.860386+0000","last_clean_scrub_stamp":"2024-09-14T08:33:30.860386+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:25:40.377777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00071425700000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":779,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-09-14T09:11:17.695468+0000","last_change":"2024-09-14T08:33:33.870929+0000","last_active":"2024-09-14T09:11:17.695468+0000","last_peered":"2024-09-14T09:11:17.695468+0000","last_clean":"2024-09-14T09:11:17.695468+0000","last_became_active":"2024-09-14T08:33:31.868362+0000","last_became_peered":"2024-09-14T08:33:31.868362+0000","last_unstale":"2024-09-14T09:11:17.695468+0000","last_undegraded":"2024-09-14T09:11:17.695468+0000","last_fullsized":"2024-09-14T09:11:17.695468+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:33:30.860386+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:33:30.860386+0000","last_clean_scrub_stamp":"2024-09-14T08:33:30.860386+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:59:01.028926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001256898,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":780,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-09-14T09:11:17.696979+0000","last_change":"2024-09-14T08:33:33.870538+0000","last_active":"2024-09-14T09:11:17.696979+0000","last_peered":"2024-09-14T09:11:17.696979+0000","last_clean":"2024-09-14T09:11:17.696979+0000","last_became_active":"2024-09-14T08:33:31.868180+0000","last_became_peered":"2024-09-14T08:33:31.868180+0000","last_unstale":"2024-09-14T09:11:17.696979+0000","last_undegraded":"2024-09-14T09:11:17.696979+0000","last_fullsized":"2024-09-14T09:11:17.696979+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:33:30.860386+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:33:30.860386+0000","last_clean_scrub_stamp":"2024-09-14T08:33:30.860386+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T19:51:08.921688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00044658000000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":780,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-09-14T09:11:17.690270+0000","last_change":"2024-09-14T08:33:33.870794+0000","last_active":"2024-09-14T09:11:17.690270+0000","last_peered":"2024-09-14T09:11:17.690270+0000","last_clean":"2024-09-14T09:11:17.690270+0000","last_became_active":"2024-09-14T08:33:31.868126+0000","last_became_peered":"2024-09-14T08:33:31.868126+0000","last_unstale":"2024-09-14T09:11:17.690270+0000","last_undegraded":"2024-09-14T09:11:17.690270+0000","last_fullsized":"2024-09-14T09:11:17.690270+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:33:30.860386+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:33:30.860386+0000","last_clean_scrub_stamp":"2024-09-14T08:33:30.860386+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:20:30.169758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064856099999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"16'2","reported_seq":782,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-09-14T09:11:18.873965+0000","last_change":"2024-09-14T08:33:33.865752+0000","last_active":"2024-09-14T09:11:18.873965+0000","last_peered":"2024-09-14T09:11:18.873965+0000","last_clean":"2024-09-14T09:11:18.873965+0000","last_became_active":"2024-09-14T08:33:31.870304+0000","last_became_peered":"2024-09-14T08:33:31.870304+0000","last_unstale":"2024-09-14T09:11:18.873965+0000","last_undegraded":"2024-09-14T09:11:18.873965+0000","last_fullsized":"2024-09-14T09:11:18.873965+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:33:30.860386+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:33:30.860386+0000","last_clean_scrub_stamp":"2024-09-14T08:33:30.860386+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:17:46.077907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000475141,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":780,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-09-14T09:11:18.026436+0000","last_change":"2024-09-14T08:33:34.137500+0000","last_active":"2024-09-14T09:11:18.026436+0000","last_peered":"2024-09-14T09:11:18.026436+0000","last_clean":"2024-09-14T09:11:18.026436+0000","last_became_active":"2024-09-14T08:33:31.869506+0000","last_became_peered":"2024-09-14T08:33:31.869506+0000","last_unstale":"2024-09-14T09:11:18.026436+0000","last_undegraded":"2024-09-14T09:11:18.026436+0000","last_fullsized":"2024-09-14T09:11:18.026436+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:33:30.860386+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:33:30.860386+0000","last_clean_scrub_stamp":"2024-09-14T08:33:30.860386+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:37:11.985802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036617900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":780,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-09-14T09:11:18.027672+0000","last_change":"2024-09-14T08:33:34.137555+0000","last_active":"2024-09-14T09:11:18.027672+0000","last_peered":"2024-09-14T09:11:18.027672+0000","last_clean":"2024-09-14T09:11:18.027672+0000","last_became_active":"2024-09-14T08:33:31.868768+0000","last_became_peered":"2024-09-14T08:33:31.868768+0000","last_unstale":"2024-09-14T09:11:18.027672+0000","last_undegraded":"2024-09-14T09:11:18.027672+0000","last_fullsized":"2024-09-14T09:11:18.027672+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:33:30.860386+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:33:30.860386+0000","last_clean_scrub_stamp":"2024-09-14T08:33:30.860386+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:16:44.486807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041116299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"14'1","reported_seq":781,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-09-14T09:11:17.693595+0000","last_change":"2024-09-14T08:33:33.870202+0000","last_active":"2024-09-14T09:11:17.693595+0000","last_peered":"2024-09-14T09:11:17.693595+0000","last_clean":"2024-09-14T09:11:17.693595+0000","last_became_active":"2024-09-14T08:33:31.868308+0000","last_became_peered":"2024-09-14T08:33:31.868308+0000","last_unstale":"2024-09-14T09:11:17.693595+0000","last_undegraded":"2024-09-14T09:11:17.693595+0000","last_fullsized":"2024-09-14T09:11:17.693595+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:33:30.860386+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:33:30.860386+0000","last_clean_scrub_stamp":"2024-09-14T08:33:30.860386+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T17:51:35.908288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00058585600000000005,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"12'52","reported_seq":846,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-09-14T09:11:17.691404+0000","last_change":"2024-09-14T08:33:29.863094+0000","last_active":"2024-09-14T09:11:17.691404+0000","last_peered":"2024-09-14T09:11:17.691404+0000","last_clean":"2024-09-14T09:11:17.691404+0000","last_became_active":"2024-09-14T08:33:29.862802+0000","last_became_peered":"2024-09-14T08:33:29.862802+0000","last_unstale":"2024-09-14T09:11:17.691404+0000","last_undegraded":"2024-09-14T09:11:17.691404+0000","last_fullsized":"2024-09-14T09:11:17.691404+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:33:28.845692+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:33:28.845692+0000","last_clean_scrub_stamp":"2024-09-14T08:33:28.845692+0000","objects_scrubbed":0,"log_size":52,"log_dups_size":0,"ondisk_log_size":52,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:35:27.535389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":79,"num_write_kb":1106,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":79,"num_write_kb":1106,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":1180736,"data_compressed":7088,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":52,"ondisk_log_size":52,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654706125,"num_pgs":80,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":829372,"kb_used_data":3112,"kb_used_omap":1,"kb_used_meta":826238,"kb_avail":93542468,"statfs":{"total":96636764160,"available":95787487232,"internally_reserved":0,"allocated":3186688,"data_stored":1699830,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":846068165},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[]},{"osd":1,"up_from":9,"seq":38654706127,"num_pgs":98,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":946548,"kb_used_data":3168,"kb_used_omap":1,"kb_used_meta":943358,"kb_avail":93425292,"statfs":{"total":96636764160,"available":95667499008,"internally_reserved":0,"allocated":3244032,"data_stored":2290198,"data_compressed":3544,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1595,"internal_metadata":965999045},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[]},{"osd":0,"up_from":9,"seq":38654706125,"num_pgs":96,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1042092,"kb_used_data":3164,"kb_used_omap":1,"kb_used_meta":1038910,"kb_avail":93329748,"statfs":{"total":96636764160,"available":95569661952,"internally_reserved":0,"allocated":3239936,"data_stored":2286132,"data_compressed":3544,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1595,"internal_metadata":1063844293},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":3544,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":3544,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T09:11:21.761 DEBUG:teuthology.orchestra.run.smithi164:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd dump --format=json 2024-09-14T09:11:22.048 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T09:11:22.048 INFO:teuthology.orchestra.run.smithi164.stdout:{"epoch":388,"fsid":"dd8fb0de-ee38-4627-a696-b631746d355c","created":"2024-09-14T08:33:15.389804+0000","modified":"2024-09-14T09:11:19.686331+0000","last_up_change":"2024-09-14T08:33:26.844101+0000","last_in_change":"2024-09-14T08:33:16.224104+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-14T08:33:28.808139+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-14T08:33:30.421146+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":16,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"93965f62-149b-4a3a-a65f-78058a6342df","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":30,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6816","nonce":2314553795},{"type":"v1","addr":"172.21.15.164:6817","nonce":2314553795}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6818","nonce":2314553795},{"type":"v1","addr":"172.21.15.164:6819","nonce":2314553795}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6822","nonce":2314553795},{"type":"v1","addr":"172.21.15.164:6823","nonce":2314553795}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6820","nonce":2314553795},{"type":"v1","addr":"172.21.15.164:6821","nonce":2314553795}]},"public_addr":"172.21.15.164:6817/2314553795","cluster_addr":"172.21.15.164:6819/2314553795","heartbeat_back_addr":"172.21.15.164:6823/2314553795","heartbeat_front_addr":"172.21.15.164:6821/2314553795","state":["exists","up"]},{"osd":1,"uuid":"dec8040a-7518-4bb9-84b5-0340c673c6bf","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6808","nonce":3507268573},{"type":"v1","addr":"172.21.15.164:6809","nonce":3507268573}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6810","nonce":3507268573},{"type":"v1","addr":"172.21.15.164:6811","nonce":3507268573}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6814","nonce":3507268573},{"type":"v1","addr":"172.21.15.164:6815","nonce":3507268573}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6812","nonce":3507268573},{"type":"v1","addr":"172.21.15.164:6813","nonce":3507268573}]},"public_addr":"172.21.15.164:6809/3507268573","cluster_addr":"172.21.15.164:6811/3507268573","heartbeat_back_addr":"172.21.15.164:6815/3507268573","heartbeat_front_addr":"172.21.15.164:6813/3507268573","state":["exists","up"]},{"osd":2,"uuid":"beca2c1e-834d-4485-b774-ece1dcd06a8d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":28,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6800","nonce":2722316831},{"type":"v1","addr":"172.21.15.164:6801","nonce":2722316831}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6802","nonce":2722316831},{"type":"v1","addr":"172.21.15.164:6803","nonce":2722316831}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6806","nonce":2722316831},{"type":"v1","addr":"172.21.15.164:6807","nonce":2722316831}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6804","nonce":2722316831},{"type":"v1","addr":"172.21.15.164:6805","nonce":2722316831}]},"public_addr":"172.21.15.164:6801/2722316831","cluster_addr":"172.21.15.164:6803/2722316831","heartbeat_back_addr":"172.21.15.164:6807/2722316831","heartbeat_front_addr":"172.21.15.164:6805/2722316831","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T08:33:22.868388+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T08:33:22.449398+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T08:33:22.240282+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-14T09:11:23.062 INFO:tasks.ceph:Scrubbing cluster2.osd.0 2024-09-14T09:11:23.062 DEBUG:teuthology.orchestra.run.smithi164:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.0 config set osd_debug_deep_scrub_sleep 0 2024-09-14T09:11:23.198 INFO:teuthology.orchestra.run.smithi164.stdout:{ 2024-09-14T09:11:23.198 INFO:teuthology.orchestra.run.smithi164.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) " 2024-09-14T09:11:23.198 INFO:teuthology.orchestra.run.smithi164.stdout:} 2024-09-14T09:11:23.209 DEBUG:teuthology.orchestra.run.smithi164:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd deep-scrub 0 2024-09-14T09:11:23.492 INFO:teuthology.orchestra.run.smithi164.stderr:instructed osd(s) 0 to deep-scrub 2024-09-14T09:11:23.505 INFO:tasks.ceph:Scrubbing cluster2.osd.1 2024-09-14T09:11:23.506 DEBUG:teuthology.orchestra.run.smithi164:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.1 config set osd_debug_deep_scrub_sleep 0 2024-09-14T09:11:23.627 INFO:teuthology.orchestra.run.smithi164.stdout:{ 2024-09-14T09:11:23.627 INFO:teuthology.orchestra.run.smithi164.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) " 2024-09-14T09:11:23.627 INFO:teuthology.orchestra.run.smithi164.stdout:} 2024-09-14T09:11:23.638 DEBUG:teuthology.orchestra.run.smithi164:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd deep-scrub 1 2024-09-14T09:11:23.922 INFO:teuthology.orchestra.run.smithi164.stderr:instructed osd(s) 1 to deep-scrub 2024-09-14T09:11:23.935 INFO:tasks.ceph:Scrubbing cluster2.osd.2 2024-09-14T09:11:23.935 DEBUG:teuthology.orchestra.run.smithi164:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.2 config set osd_debug_deep_scrub_sleep 0 2024-09-14T09:11:24.056 INFO:teuthology.orchestra.run.smithi164.stdout:{ 2024-09-14T09:11:24.057 INFO:teuthology.orchestra.run.smithi164.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) " 2024-09-14T09:11:24.057 INFO:teuthology.orchestra.run.smithi164.stdout:} 2024-09-14T09:11:24.067 DEBUG:teuthology.orchestra.run.smithi164:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd deep-scrub 2 2024-09-14T09:11:24.353 INFO:teuthology.orchestra.run.smithi164.stderr:instructed osd(s) 2 to deep-scrub 2024-09-14T09:11:24.366 DEBUG:teuthology.orchestra.run.smithi164:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2024-09-14T09:11:24.648 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T09:11:24.648 INFO:teuthology.orchestra.run.smithi164.stderr:dumped all 2024-09-14T09:11:24.660 INFO:teuthology.orchestra.run.smithi164.stdout:{"pg_ready":true,"pg_map":{"version":1524,"stamp":"2024-09-14T09:11:23.381850+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":81,"num_write_kb":1108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":274,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":2818012,"kb_used_data":9444,"kb_used_omap":4,"kb_used_meta":2808507,"kb_avail":280297508,"statfs":{"total":289910292480,"available":287024648192,"internally_reserved":0,"allocated":9670656,"data_stored":6276160,"data_compressed":7088,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":4785,"internal_metadata":2875911503},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":15,"apply_latency_ms":15,"commit_latency_ns":15000000,"apply_latency_ns":15000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-536870966,"num_objects":-164,"num_object_clones":0,"num_object_copies":-328,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-164,"num_whiteouts":0,"num_read":-115387,"num_read_kb":-39629441,"num_write":-406691,"num_write_kb":-148835173,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-7,"num_bytes_recovered":-12582931,"num_keys_recovered":0,"num_objects_omap":-14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.001706"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":779,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-09-14T09:11:17.691832+0000","last_change":"2024-09-14T08:33:33.870517+0000","last_active":"2024-09-14T09:11:17.691832+0000","last_peered":"2024-09-14T09:11:17.691832+0000","last_clean":"2024-09-14T09:11:17.691832+0000","last_became_active":"2024-09-14T08:33:31.868938+0000","last_became_peered":"2024-09-14T08:33:31.868938+0000","last_unstale":"2024-09-14T09:11:17.691832+0000","last_undegraded":"2024-09-14T09:11:17.691832+0000","last_fullsized":"2024-09-14T09:11:17.691832+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:33:30.860386+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:33:30.860386+0000","last_clean_scrub_stamp":"2024-09-14T08:33:30.860386+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:25:40.377777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00071425700000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":779,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-09-14T09:11:17.695468+0000","last_change":"2024-09-14T08:33:33.870929+0000","last_active":"2024-09-14T09:11:17.695468+0000","last_peered":"2024-09-14T09:11:17.695468+0000","last_clean":"2024-09-14T09:11:17.695468+0000","last_became_active":"2024-09-14T08:33:31.868362+0000","last_became_peered":"2024-09-14T08:33:31.868362+0000","last_unstale":"2024-09-14T09:11:17.695468+0000","last_undegraded":"2024-09-14T09:11:17.695468+0000","last_fullsized":"2024-09-14T09:11:17.695468+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:33:30.860386+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:33:30.860386+0000","last_clean_scrub_stamp":"2024-09-14T08:33:30.860386+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:59:01.028926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001256898,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":780,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-09-14T09:11:17.696979+0000","last_change":"2024-09-14T08:33:33.870538+0000","last_active":"2024-09-14T09:11:17.696979+0000","last_peered":"2024-09-14T09:11:17.696979+0000","last_clean":"2024-09-14T09:11:17.696979+0000","last_became_active":"2024-09-14T08:33:31.868180+0000","last_became_peered":"2024-09-14T08:33:31.868180+0000","last_unstale":"2024-09-14T09:11:17.696979+0000","last_undegraded":"2024-09-14T09:11:17.696979+0000","last_fullsized":"2024-09-14T09:11:17.696979+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:33:30.860386+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:33:30.860386+0000","last_clean_scrub_stamp":"2024-09-14T08:33:30.860386+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T19:51:08.921688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00044658000000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":780,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-09-14T09:11:17.690270+0000","last_change":"2024-09-14T08:33:33.870794+0000","last_active":"2024-09-14T09:11:17.690270+0000","last_peered":"2024-09-14T09:11:17.690270+0000","last_clean":"2024-09-14T09:11:17.690270+0000","last_became_active":"2024-09-14T08:33:31.868126+0000","last_became_peered":"2024-09-14T08:33:31.868126+0000","last_unstale":"2024-09-14T09:11:17.690270+0000","last_undegraded":"2024-09-14T09:11:17.690270+0000","last_fullsized":"2024-09-14T09:11:17.690270+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:33:30.860386+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:33:30.860386+0000","last_clean_scrub_stamp":"2024-09-14T08:33:30.860386+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:20:30.169758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064856099999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"16'2","reported_seq":782,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-09-14T09:11:18.873965+0000","last_change":"2024-09-14T08:33:33.865752+0000","last_active":"2024-09-14T09:11:18.873965+0000","last_peered":"2024-09-14T09:11:18.873965+0000","last_clean":"2024-09-14T09:11:18.873965+0000","last_became_active":"2024-09-14T08:33:31.870304+0000","last_became_peered":"2024-09-14T08:33:31.870304+0000","last_unstale":"2024-09-14T09:11:18.873965+0000","last_undegraded":"2024-09-14T09:11:18.873965+0000","last_fullsized":"2024-09-14T09:11:18.873965+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:33:30.860386+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:33:30.860386+0000","last_clean_scrub_stamp":"2024-09-14T08:33:30.860386+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:17:46.077907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000475141,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":780,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-09-14T09:11:18.026436+0000","last_change":"2024-09-14T08:33:34.137500+0000","last_active":"2024-09-14T09:11:18.026436+0000","last_peered":"2024-09-14T09:11:18.026436+0000","last_clean":"2024-09-14T09:11:18.026436+0000","last_became_active":"2024-09-14T08:33:31.869506+0000","last_became_peered":"2024-09-14T08:33:31.869506+0000","last_unstale":"2024-09-14T09:11:18.026436+0000","last_undegraded":"2024-09-14T09:11:18.026436+0000","last_fullsized":"2024-09-14T09:11:18.026436+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:33:30.860386+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:33:30.860386+0000","last_clean_scrub_stamp":"2024-09-14T08:33:30.860386+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:37:11.985802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036617900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":780,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-09-14T09:11:18.027672+0000","last_change":"2024-09-14T08:33:34.137555+0000","last_active":"2024-09-14T09:11:18.027672+0000","last_peered":"2024-09-14T09:11:18.027672+0000","last_clean":"2024-09-14T09:11:18.027672+0000","last_became_active":"2024-09-14T08:33:31.868768+0000","last_became_peered":"2024-09-14T08:33:31.868768+0000","last_unstale":"2024-09-14T09:11:18.027672+0000","last_undegraded":"2024-09-14T09:11:18.027672+0000","last_fullsized":"2024-09-14T09:11:18.027672+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:33:30.860386+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:33:30.860386+0000","last_clean_scrub_stamp":"2024-09-14T08:33:30.860386+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:16:44.486807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041116299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"14'1","reported_seq":781,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-09-14T09:11:17.693595+0000","last_change":"2024-09-14T08:33:33.870202+0000","last_active":"2024-09-14T09:11:17.693595+0000","last_peered":"2024-09-14T09:11:17.693595+0000","last_clean":"2024-09-14T09:11:17.693595+0000","last_became_active":"2024-09-14T08:33:31.868308+0000","last_became_peered":"2024-09-14T08:33:31.868308+0000","last_unstale":"2024-09-14T09:11:17.693595+0000","last_undegraded":"2024-09-14T09:11:17.693595+0000","last_fullsized":"2024-09-14T09:11:17.693595+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:33:30.860386+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:33:30.860386+0000","last_clean_scrub_stamp":"2024-09-14T08:33:30.860386+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T17:51:35.908288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00058585600000000005,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"12'52","reported_seq":846,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-09-14T09:11:17.691404+0000","last_change":"2024-09-14T08:33:29.863094+0000","last_active":"2024-09-14T09:11:17.691404+0000","last_peered":"2024-09-14T09:11:17.691404+0000","last_clean":"2024-09-14T09:11:17.691404+0000","last_became_active":"2024-09-14T08:33:29.862802+0000","last_became_peered":"2024-09-14T08:33:29.862802+0000","last_unstale":"2024-09-14T09:11:17.691404+0000","last_undegraded":"2024-09-14T09:11:17.691404+0000","last_fullsized":"2024-09-14T09:11:17.691404+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:33:28.845692+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:33:28.845692+0000","last_clean_scrub_stamp":"2024-09-14T08:33:28.845692+0000","objects_scrubbed":0,"log_size":52,"log_dups_size":0,"ondisk_log_size":52,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:35:27.535389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":79,"num_write_kb":1106,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":79,"num_write_kb":1106,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":1180736,"data_compressed":7088,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":52,"ondisk_log_size":52,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654706125,"num_pgs":80,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":829372,"kb_used_data":3112,"kb_used_omap":1,"kb_used_meta":826238,"kb_avail":93542468,"statfs":{"total":96636764160,"available":95787487232,"internally_reserved":0,"allocated":3186688,"data_stored":1699830,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":846068165},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[]},{"osd":1,"up_from":9,"seq":38654706127,"num_pgs":98,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":946548,"kb_used_data":3168,"kb_used_omap":1,"kb_used_meta":943358,"kb_avail":93425292,"statfs":{"total":96636764160,"available":95667499008,"internally_reserved":0,"allocated":3244032,"data_stored":2290198,"data_compressed":3544,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1595,"internal_metadata":965999045},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[]},{"osd":0,"up_from":9,"seq":38654706125,"num_pgs":96,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1042092,"kb_used_data":3164,"kb_used_omap":1,"kb_used_meta":1038910,"kb_avail":93329748,"statfs":{"total":96636764160,"available":95569661952,"internally_reserved":0,"allocated":3239936,"data_stored":2286132,"data_compressed":3544,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1595,"internal_metadata":1063844293},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":3544,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":3544,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T09:11:24.664 INFO:tasks.ceph:pgid 2.7 last_scrub_stamp 2024-09-14T08:33:30.860386+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=14, tm_hour=8, tm_min=33, tm_sec=30, tm_wday=5, tm_yday=258, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=14, tm_hour=9, tm_min=11, tm_sec=22, tm_wday=5, tm_yday=258, tm_isdst=0) 2024-09-14T09:11:24.664 INFO:tasks.ceph:pgid 2.6 last_scrub_stamp 2024-09-14T08:33:30.860386+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=14, tm_hour=8, tm_min=33, tm_sec=30, tm_wday=5, tm_yday=258, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=14, tm_hour=9, tm_min=11, tm_sec=22, tm_wday=5, tm_yday=258, tm_isdst=0) 2024-09-14T09:11:24.664 INFO:tasks.ceph:pgid 2.5 last_scrub_stamp 2024-09-14T08:33:30.860386+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=14, tm_hour=8, tm_min=33, tm_sec=30, tm_wday=5, tm_yday=258, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=14, tm_hour=9, tm_min=11, tm_sec=22, tm_wday=5, tm_yday=258, tm_isdst=0) 2024-09-14T09:11:24.664 INFO:tasks.ceph:pgid 2.4 last_scrub_stamp 2024-09-14T08:33:30.860386+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=14, tm_hour=8, tm_min=33, tm_sec=30, tm_wday=5, tm_yday=258, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=14, tm_hour=9, tm_min=11, tm_sec=22, tm_wday=5, tm_yday=258, tm_isdst=0) 2024-09-14T09:11:24.664 INFO:tasks.ceph:pgid 2.2 last_scrub_stamp 2024-09-14T08:33:30.860386+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=14, tm_hour=8, tm_min=33, tm_sec=30, tm_wday=5, tm_yday=258, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=14, tm_hour=9, tm_min=11, tm_sec=22, tm_wday=5, tm_yday=258, tm_isdst=0) 2024-09-14T09:11:24.664 INFO:tasks.ceph:pgid 2.1 last_scrub_stamp 2024-09-14T08:33:30.860386+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=14, tm_hour=8, tm_min=33, tm_sec=30, tm_wday=5, tm_yday=258, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=14, tm_hour=9, tm_min=11, tm_sec=22, tm_wday=5, tm_yday=258, tm_isdst=0) 2024-09-14T09:11:24.665 INFO:tasks.ceph:pgid 2.0 last_scrub_stamp 2024-09-14T08:33:30.860386+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=14, tm_hour=8, tm_min=33, tm_sec=30, tm_wday=5, tm_yday=258, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=14, tm_hour=9, tm_min=11, tm_sec=22, tm_wday=5, tm_yday=258, tm_isdst=0) 2024-09-14T09:11:24.665 INFO:tasks.ceph:pgid 2.3 last_scrub_stamp 2024-09-14T08:33:30.860386+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=14, tm_hour=8, tm_min=33, tm_sec=30, tm_wday=5, tm_yday=258, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=14, tm_hour=9, tm_min=11, tm_sec=22, tm_wday=5, tm_yday=258, tm_isdst=0) 2024-09-14T09:11:24.665 INFO:tasks.ceph:pgid 1.0 last_scrub_stamp 2024-09-14T08:33:28.845692+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=14, tm_hour=8, tm_min=33, tm_sec=28, tm_wday=5, tm_yday=258, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=14, tm_hour=9, tm_min=11, tm_sec=22, tm_wday=5, tm_yday=258, tm_isdst=0) 2024-09-14T09:11:24.665 INFO:tasks.ceph:Still waiting for all pgs to be scrubbed. 2024-09-14T09:11:44.666 DEBUG:teuthology.orchestra.run.smithi164:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2024-09-14T09:11:44.956 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-14T09:11:44.957 INFO:teuthology.orchestra.run.smithi164.stderr:dumped all 2024-09-14T09:11:44.969 INFO:teuthology.orchestra.run.smithi164.stdout:{"pg_ready":true,"pg_map":{"version":1534,"stamp":"2024-09-14T09:11:43.385301+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":81,"num_write_kb":1108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":59,"ondisk_log_size":59,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":2836088,"kb_used_data":9480,"kb_used_omap":4,"kb_used_meta":2826555,"kb_avail":280279432,"statfs":{"total":289910292480,"available":287006138368,"internally_reserved":0,"allocated":9707520,"data_stored":6291097,"data_compressed":7088,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":4785,"internal_metadata":2894392655},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002192"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":789,"reported_epoch":388,"state":"active+clean","last_fresh":"2024-09-14T09:11:29.097511+0000","last_change":"2024-09-14T09:11:29.097511+0000","last_active":"2024-09-14T09:11:29.097511+0000","last_peered":"2024-09-14T09:11:29.097511+0000","last_clean":"2024-09-14T09:11:29.097511+0000","last_became_active":"2024-09-14T08:33:31.868938+0000","last_became_peered":"2024-09-14T08:33:31.868938+0000","last_unstale":"2024-09-14T09:11:29.097511+0000","last_undegraded":"2024-09-14T09:11:29.097511+0000","last_fullsized":"2024-09-14T09:11:29.097511+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T09:11:29.097426+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T09:11:29.097426+0000","last_clean_scrub_stamp":"2024-09-14T09:11:29.097426+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:45:54.768440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00071425700000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":789,"reported_epoch":388,"state":"active+clean","last_fresh":"2024-09-14T09:11:26.101374+0000","last_change":"2024-09-14T09:11:26.101374+0000","last_active":"2024-09-14T09:11:26.101374+0000","last_peered":"2024-09-14T09:11:26.101374+0000","last_clean":"2024-09-14T09:11:26.101374+0000","last_became_active":"2024-09-14T08:33:31.868362+0000","last_became_peered":"2024-09-14T08:33:31.868362+0000","last_unstale":"2024-09-14T09:11:26.101374+0000","last_undegraded":"2024-09-14T09:11:26.101374+0000","last_fullsized":"2024-09-14T09:11:26.101374+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T09:11:26.101287+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T09:11:26.101287+0000","last_clean_scrub_stamp":"2024-09-14T09:11:26.101287+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:48:52.372464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001256898,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":790,"reported_epoch":388,"state":"active+clean","last_fresh":"2024-09-14T09:11:27.127043+0000","last_change":"2024-09-14T09:11:27.127043+0000","last_active":"2024-09-14T09:11:27.127043+0000","last_peered":"2024-09-14T09:11:27.127043+0000","last_clean":"2024-09-14T09:11:27.127043+0000","last_became_active":"2024-09-14T08:33:31.868180+0000","last_became_peered":"2024-09-14T08:33:31.868180+0000","last_unstale":"2024-09-14T09:11:27.127043+0000","last_undegraded":"2024-09-14T09:11:27.127043+0000","last_fullsized":"2024-09-14T09:11:27.127043+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T09:11:27.126944+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T09:11:27.126944+0000","last_clean_scrub_stamp":"2024-09-14T09:11:27.126944+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T20:51:23.587907+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0.00044658000000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":790,"reported_epoch":388,"state":"active+clean","last_fresh":"2024-09-14T09:11:25.130786+0000","last_change":"2024-09-14T09:11:25.130786+0000","last_active":"2024-09-14T09:11:25.130786+0000","last_peered":"2024-09-14T09:11:25.130786+0000","last_clean":"2024-09-14T09:11:25.130786+0000","last_became_active":"2024-09-14T08:33:31.868126+0000","last_became_peered":"2024-09-14T08:33:31.868126+0000","last_unstale":"2024-09-14T09:11:25.130786+0000","last_undegraded":"2024-09-14T09:11:25.130786+0000","last_fullsized":"2024-09-14T09:11:25.130786+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T09:11:25.130717+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T09:11:25.130717+0000","last_clean_scrub_stamp":"2024-09-14T09:11:25.130717+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T18:04:08.137488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064856099999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"388'3","reported_seq":793,"reported_epoch":388,"state":"active+clean","last_fresh":"2024-09-14T09:11:23.810407+0000","last_change":"2024-09-14T09:11:23.810407+0000","last_active":"2024-09-14T09:11:23.810407+0000","last_peered":"2024-09-14T09:11:23.810407+0000","last_clean":"2024-09-14T09:11:23.810407+0000","last_became_active":"2024-09-14T08:33:31.870304+0000","last_became_peered":"2024-09-14T08:33:31.870304+0000","last_unstale":"2024-09-14T09:11:23.810407+0000","last_undegraded":"2024-09-14T09:11:23.810407+0000","last_fullsized":"2024-09-14T09:11:23.810407+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"388'3","last_scrub_stamp":"2024-09-14T09:11:23.810310+0000","last_deep_scrub":"388'3","last_deep_scrub_stamp":"2024-09-14T09:11:23.810310+0000","last_clean_scrub_stamp":"2024-09-14T09:11:23.810310+0000","objects_scrubbed":1,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T16:29:44.834492+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0.000475141,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":790,"reported_epoch":388,"state":"active+clean","last_fresh":"2024-09-14T09:11:25.884673+0000","last_change":"2024-09-14T09:11:25.884673+0000","last_active":"2024-09-14T09:11:25.884673+0000","last_peered":"2024-09-14T09:11:25.884673+0000","last_clean":"2024-09-14T09:11:25.884673+0000","last_became_active":"2024-09-14T08:33:31.869506+0000","last_became_peered":"2024-09-14T08:33:31.869506+0000","last_unstale":"2024-09-14T09:11:25.884673+0000","last_undegraded":"2024-09-14T09:11:25.884673+0000","last_fullsized":"2024-09-14T09:11:25.884673+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T09:11:25.884544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T09:11:25.884544+0000","last_clean_scrub_stamp":"2024-09-14T09:11:25.884544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:14:49.465438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036617900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":791,"reported_epoch":388,"state":"active+clean","last_fresh":"2024-09-14T09:11:24.886631+0000","last_change":"2024-09-14T09:11:24.886572+0000","last_active":"2024-09-14T09:11:24.886631+0000","last_peered":"2024-09-14T09:11:24.886631+0000","last_clean":"2024-09-14T09:11:24.886631+0000","last_became_active":"2024-09-14T08:33:31.868768+0000","last_became_peered":"2024-09-14T08:33:31.868768+0000","last_unstale":"2024-09-14T09:11:24.886631+0000","last_undegraded":"2024-09-14T09:11:24.886631+0000","last_fullsized":"2024-09-14T09:11:24.886631+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T09:11:24.886501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T09:11:24.886501+0000","last_clean_scrub_stamp":"2024-09-14T09:11:24.886501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:42:39.238706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041116299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"388'2","reported_seq":792,"reported_epoch":388,"state":"active+clean","last_fresh":"2024-09-14T09:11:24.147304+0000","last_change":"2024-09-14T09:11:24.147304+0000","last_active":"2024-09-14T09:11:24.147304+0000","last_peered":"2024-09-14T09:11:24.147304+0000","last_clean":"2024-09-14T09:11:24.147304+0000","last_became_active":"2024-09-14T08:33:31.868308+0000","last_became_peered":"2024-09-14T08:33:31.868308+0000","last_unstale":"2024-09-14T09:11:24.147304+0000","last_undegraded":"2024-09-14T09:11:24.147304+0000","last_fullsized":"2024-09-14T09:11:24.147304+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"388'2","last_scrub_stamp":"2024-09-14T09:11:24.147207+0000","last_deep_scrub":"388'2","last_deep_scrub_stamp":"2024-09-14T09:11:24.147207+0000","last_clean_scrub_stamp":"2024-09-14T09:11:24.147207+0000","objects_scrubbed":1,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:02:02.663759+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0.00058585600000000005,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"388'54","reported_seq":859,"reported_epoch":388,"state":"active+clean","last_fresh":"2024-09-14T09:11:28.126113+0000","last_change":"2024-09-14T09:11:28.126035+0000","last_active":"2024-09-14T09:11:28.126113+0000","last_peered":"2024-09-14T09:11:28.126113+0000","last_clean":"2024-09-14T09:11:28.126113+0000","last_became_active":"2024-09-14T08:33:29.862802+0000","last_became_peered":"2024-09-14T08:33:29.862802+0000","last_unstale":"2024-09-14T09:11:28.126113+0000","last_undegraded":"2024-09-14T09:11:28.126113+0000","last_fullsized":"2024-09-14T09:11:28.126113+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"388'54","last_scrub_stamp":"2024-09-14T09:11:28.125939+0000","last_deep_scrub":"388'54","last_deep_scrub_stamp":"2024-09-14T09:11:28.125939+0000","last_clean_scrub_stamp":"2024-09-14T09:11:28.125939+0000","objects_scrubbed":2,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T20:51:03.736039+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":79,"num_write_kb":1106,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":5,"ondisk_log_size":5,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":79,"num_write_kb":1106,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":1180736,"data_compressed":7088,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":54,"ondisk_log_size":54,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654706129,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":829376,"kb_used_data":3120,"kb_used_omap":1,"kb_used_meta":826238,"kb_avail":93542464,"statfs":{"total":96636764160,"available":95787483136,"internally_reserved":0,"allocated":3194880,"data_stored":1703441,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":846068165},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":9,"seq":38654706131,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":946556,"kb_used_data":3180,"kb_used_omap":1,"kb_used_meta":943358,"kb_avail":93425284,"statfs":{"total":96636764160,"available":95667490816,"internally_reserved":0,"allocated":3256320,"data_stored":2293828,"data_compressed":3544,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1595,"internal_metadata":965999045},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":9,"seq":38654706129,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1060156,"kb_used_data":3180,"kb_used_omap":1,"kb_used_meta":1056958,"kb_avail":93311684,"statfs":{"total":96636764160,"available":95551164416,"internally_reserved":0,"allocated":3256320,"data_stored":2293828,"data_compressed":3544,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1595,"internal_metadata":1082325445},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":3544,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":3544,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T09:11:44.970 DEBUG:teuthology.orchestra.run.smithi164:> sudo ceph --cluster cluster2 config set global mon_health_to_clog false 2024-09-14T09:11:45.271 INFO:teuthology.misc:Shutting down mds daemons... 2024-09-14T09:11:45.272 INFO:teuthology.misc:Shutting down osd daemons... 2024-09-14T09:11:45.272 DEBUG:tasks.ceph.cluster2.osd.0:waiting for process to exit 2024-09-14T09:11:45.272 INFO:teuthology.orchestra.run:waiting for 300 2024-09-14T09:11:45.334 INFO:tasks.ceph.cluster2.osd.0:Stopped 2024-09-14T09:11:45.334 DEBUG:tasks.ceph.cluster2.osd.1:waiting for process to exit 2024-09-14T09:11:45.334 INFO:teuthology.orchestra.run:waiting for 300 2024-09-14T09:11:45.395 INFO:tasks.ceph.cluster2.osd.1:Stopped 2024-09-14T09:11:45.395 DEBUG:tasks.ceph.cluster2.osd.2:waiting for process to exit 2024-09-14T09:11:45.395 INFO:teuthology.orchestra.run:waiting for 300 2024-09-14T09:11:45.454 INFO:tasks.ceph.cluster2.osd.2:Stopped 2024-09-14T09:11:45.454 INFO:teuthology.misc:Shutting down mgr daemons... 2024-09-14T09:11:45.454 DEBUG:tasks.ceph.cluster2.mgr.x:waiting for process to exit 2024-09-14T09:11:45.455 INFO:teuthology.orchestra.run:waiting for 300 2024-09-14T09:11:45.483 INFO:tasks.ceph.cluster2.mgr.x:Stopped 2024-09-14T09:11:45.483 INFO:teuthology.misc:Shutting down mon daemons... 2024-09-14T09:11:45.484 DEBUG:tasks.ceph.cluster2.mon.a:waiting for process to exit 2024-09-14T09:11:45.484 INFO:teuthology.orchestra.run:waiting for 300 2024-09-14T09:11:45.496 INFO:tasks.ceph.cluster2.mon.a:Stopped 2024-09-14T09:11:45.496 INFO:tasks.ceph:Checking cluster log for badness... 2024-09-14T09:11:45.497 DEBUG:teuthology.orchestra.run.smithi164:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/cluster2.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v '\(OSD_SLOW_PING_TIME' | head -n 1 2024-09-14T09:11:45.564 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster2-0 on ubuntu@smithi164.front.sepia.ceph.com 2024-09-14T09:11:45.565 DEBUG:teuthology.orchestra.run.smithi164:> sync && sudo umount -f /var/lib/ceph/osd/cluster2-0 2024-09-14T09:11:45.986 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster2-1 on ubuntu@smithi164.front.sepia.ceph.com 2024-09-14T09:11:45.986 DEBUG:teuthology.orchestra.run.smithi164:> sync && sudo umount -f /var/lib/ceph/osd/cluster2-1 2024-09-14T09:11:46.095 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster2-2 on ubuntu@smithi164.front.sepia.ceph.com 2024-09-14T09:11:46.095 DEBUG:teuthology.orchestra.run.smithi164:> sync && sudo umount -f /var/lib/ceph/osd/cluster2-2 2024-09-14T09:11:46.197 INFO:tasks.ceph:Archiving mon data... 2024-09-14T09:11:46.199 DEBUG:teuthology.misc:Transferring archived files from smithi164:/var/lib/ceph/mon/cluster2-a to /home/teuthworker/archive/teuthology-2024-09-10_20:16:17-rbd-main-distro-default-smithi/7900492/data/cluster2.mon.a.tgz 2024-09-14T09:11:46.199 DEBUG:teuthology.orchestra.run.smithi164:> mktemp 2024-09-14T09:11:46.214 INFO:teuthology.orchestra.run.smithi164.stdout:/tmp/tmp.loM2ZbToja 2024-09-14T09:11:46.214 DEBUG:teuthology.orchestra.run.smithi164:> sudo tar cz -f - -C /var/lib/ceph/mon/cluster2-a -- . > /tmp/tmp.loM2ZbToja 2024-09-14T09:11:46.404 DEBUG:teuthology.orchestra.run.smithi164:> sudo chmod 0666 /tmp/tmp.loM2ZbToja 2024-09-14T09:11:46.488 DEBUG:teuthology.orchestra.remote:smithi164:/tmp/tmp.loM2ZbToja is 412KB 2024-09-14T09:11:46.586 DEBUG:teuthology.orchestra.run.smithi164:> rm -fr /tmp/tmp.loM2ZbToja 2024-09-14T09:11:46.601 INFO:tasks.ceph:Cleaning ceph cluster... 2024-09-14T09:11:46.601 DEBUG:teuthology.orchestra.run.smithi135:> sudo rm -rf -- /etc/ceph/cluster2.conf /etc/ceph/cluster2.keyring /home/ubuntu/cephtest/cluster2.data /home/ubuntu/cephtest/cluster2.monmap /home/ubuntu/cephtest/../*.pid 2024-09-14T09:11:46.604 DEBUG:teuthology.orchestra.run.smithi164:> sudo rm -rf -- /etc/ceph/cluster2.conf /etc/ceph/cluster2.keyring /home/ubuntu/cephtest/cluster2.data /home/ubuntu/cephtest/cluster2.monmap /home/ubuntu/cephtest/../*.pid 2024-09-14T09:11:46.665 DEBUG:teuthology.run_tasks:Unwinding manager ceph 2024-09-14T09:11:46.677 INFO:tasks.ceph.ceph_manager.cluster1:waiting for clean 2024-09-14T09:11:46.677 DEBUG:teuthology.orchestra.run.smithi135:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2024-09-14T09:11:46.968 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T09:11:46.968 INFO:teuthology.orchestra.run.smithi135.stderr:dumped all 2024-09-14T09:11:46.980 INFO:teuthology.orchestra.run.smithi135.stdout:{"pg_ready":true,"pg_map":{"version":1632,"stamp":"2024-09-14T09:11:45.203281+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":81,"num_write_kb":1108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":293564,"kb_used_data":11328,"kb_used_omap":4,"kb_used_meta":282171,"kb_avail":282821956,"statfs":{"total":289910292480,"available":289609682944,"internally_reserved":0,"allocated":11599872,"data_stored":7316431,"data_compressed":7092,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":4785,"internal_metadata":288943439},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002409"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":934,"reported_epoch":465,"state":"active+clean","last_fresh":"2024-09-14T09:11:18.565126+0000","last_change":"2024-09-14T08:32:54.696724+0000","last_active":"2024-09-14T09:11:18.565126+0000","last_peered":"2024-09-14T09:11:18.565126+0000","last_clean":"2024-09-14T09:11:18.565126+0000","last_became_active":"2024-09-14T08:32:52.693987+0000","last_became_peered":"2024-09-14T08:32:52.693987+0000","last_unstale":"2024-09-14T09:11:18.565126+0000","last_undegraded":"2024-09-14T09:11:18.565126+0000","last_fullsized":"2024-09-14T09:11:18.565126+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:32:51.683516+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:32:51.683516+0000","last_clean_scrub_stamp":"2024-09-14T08:32:51.683516+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T17:10:29.729916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00111144,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":934,"reported_epoch":465,"state":"active+clean","last_fresh":"2024-09-14T09:11:18.565627+0000","last_change":"2024-09-14T08:32:54.696686+0000","last_active":"2024-09-14T09:11:18.565627+0000","last_peered":"2024-09-14T09:11:18.565627+0000","last_clean":"2024-09-14T09:11:18.565627+0000","last_became_active":"2024-09-14T08:32:52.691575+0000","last_became_peered":"2024-09-14T08:32:52.691575+0000","last_unstale":"2024-09-14T09:11:18.565627+0000","last_undegraded":"2024-09-14T09:11:18.565627+0000","last_fullsized":"2024-09-14T09:11:18.565627+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:32:51.683516+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:32:51.683516+0000","last_clean_scrub_stamp":"2024-09-14T08:32:51.683516+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T18:10:27.710040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001078809,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":935,"reported_epoch":465,"state":"active+clean","last_fresh":"2024-09-14T09:11:18.563618+0000","last_change":"2024-09-14T08:32:54.696401+0000","last_active":"2024-09-14T09:11:18.563618+0000","last_peered":"2024-09-14T09:11:18.563618+0000","last_clean":"2024-09-14T09:11:18.563618+0000","last_became_active":"2024-09-14T08:32:52.694100+0000","last_became_peered":"2024-09-14T08:32:52.694100+0000","last_unstale":"2024-09-14T09:11:18.563618+0000","last_undegraded":"2024-09-14T09:11:18.563618+0000","last_fullsized":"2024-09-14T09:11:18.563618+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:32:51.683516+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:32:51.683516+0000","last_clean_scrub_stamp":"2024-09-14T08:32:51.683516+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:48:32.158959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00082548099999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":935,"reported_epoch":465,"state":"active+clean","last_fresh":"2024-09-14T09:11:18.562068+0000","last_change":"2024-09-14T08:32:54.696398+0000","last_active":"2024-09-14T09:11:18.562068+0000","last_peered":"2024-09-14T09:11:18.562068+0000","last_clean":"2024-09-14T09:11:18.562068+0000","last_became_active":"2024-09-14T08:32:52.692903+0000","last_became_peered":"2024-09-14T08:32:52.692903+0000","last_unstale":"2024-09-14T09:11:18.562068+0000","last_undegraded":"2024-09-14T09:11:18.562068+0000","last_fullsized":"2024-09-14T09:11:18.562068+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:32:51.683516+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:32:51.683516+0000","last_clean_scrub_stamp":"2024-09-14T08:32:51.683516+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T20:22:32.946236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00085427300000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"16'2","reported_seq":937,"reported_epoch":465,"state":"active+clean","last_fresh":"2024-09-14T09:11:18.561824+0000","last_change":"2024-09-14T08:32:54.692785+0000","last_active":"2024-09-14T09:11:18.561824+0000","last_peered":"2024-09-14T09:11:18.561824+0000","last_clean":"2024-09-14T09:11:18.561824+0000","last_became_active":"2024-09-14T08:32:52.691649+0000","last_became_peered":"2024-09-14T08:32:52.691649+0000","last_unstale":"2024-09-14T09:11:18.561824+0000","last_undegraded":"2024-09-14T09:11:18.561824+0000","last_fullsized":"2024-09-14T09:11:18.561824+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:32:51.683516+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:32:51.683516+0000","last_clean_scrub_stamp":"2024-09-14T08:32:51.683516+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:27:21.636753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00094884699999999995,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":936,"reported_epoch":465,"state":"active+clean","last_fresh":"2024-09-14T09:11:18.549100+0000","last_change":"2024-09-14T08:32:54.689445+0000","last_active":"2024-09-14T09:11:18.549100+0000","last_peered":"2024-09-14T09:11:18.549100+0000","last_clean":"2024-09-14T09:11:18.549100+0000","last_became_active":"2024-09-14T08:32:52.693218+0000","last_became_peered":"2024-09-14T08:32:52.693218+0000","last_unstale":"2024-09-14T09:11:18.549100+0000","last_undegraded":"2024-09-14T09:11:18.549100+0000","last_fullsized":"2024-09-14T09:11:18.549100+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:32:51.683516+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:32:51.683516+0000","last_clean_scrub_stamp":"2024-09-14T08:32:51.683516+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T17:13:15.903145+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000454201,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":937,"reported_epoch":465,"state":"active+clean","last_fresh":"2024-09-14T09:11:18.549034+0000","last_change":"2024-09-14T08:32:54.689408+0000","last_active":"2024-09-14T09:11:18.549034+0000","last_peered":"2024-09-14T09:11:18.549034+0000","last_clean":"2024-09-14T09:11:18.549034+0000","last_became_active":"2024-09-14T08:32:52.693247+0000","last_became_peered":"2024-09-14T08:32:52.693247+0000","last_unstale":"2024-09-14T09:11:18.549034+0000","last_undegraded":"2024-09-14T09:11:18.549034+0000","last_fullsized":"2024-09-14T09:11:18.549034+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:32:51.683516+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:32:51.683516+0000","last_clean_scrub_stamp":"2024-09-14T08:32:51.683516+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:29:38.741547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00044293799999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"14'1","reported_seq":936,"reported_epoch":465,"state":"active+clean","last_fresh":"2024-09-14T09:11:18.562325+0000","last_change":"2024-09-14T08:32:54.697190+0000","last_active":"2024-09-14T09:11:18.562325+0000","last_peered":"2024-09-14T09:11:18.562325+0000","last_clean":"2024-09-14T09:11:18.562325+0000","last_became_active":"2024-09-14T08:32:52.691607+0000","last_became_peered":"2024-09-14T08:32:52.691607+0000","last_unstale":"2024-09-14T09:11:18.562325+0000","last_undegraded":"2024-09-14T09:11:18.562325+0000","last_fullsized":"2024-09-14T09:11:18.562325+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:32:51.683516+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:32:51.683516+0000","last_clean_scrub_stamp":"2024-09-14T08:32:51.683516+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T16:28:20.832608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0014408509999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"12'52","reported_seq":1002,"reported_epoch":465,"state":"active+clean","last_fresh":"2024-09-14T09:11:18.564234+0000","last_change":"2024-09-14T08:32:51.096141+0000","last_active":"2024-09-14T09:11:18.564234+0000","last_peered":"2024-09-14T09:11:18.564234+0000","last_clean":"2024-09-14T09:11:18.564234+0000","last_became_active":"2024-09-14T08:32:51.095164+0000","last_became_peered":"2024-09-14T08:32:51.095164+0000","last_unstale":"2024-09-14T09:11:18.564234+0000","last_undegraded":"2024-09-14T09:11:18.564234+0000","last_fullsized":"2024-09-14T09:11:18.564234+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:32:49.668980+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:32:49.668980+0000","last_clean_scrub_stamp":"2024-09-14T08:32:49.668980+0000","objects_scrubbed":0,"log_size":52,"log_dups_size":0,"ondisk_log_size":52,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T19:52:09.318667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":79,"num_write_kb":1106,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":79,"num_write_kb":1106,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":1180736,"data_compressed":7092,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":52,"ondisk_log_size":52,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654706137,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":89004,"kb_used_data":3736,"kb_used_omap":1,"kb_used_meta":85246,"kb_avail":94282836,"statfs":{"total":96636764160,"available":96545624064,"internally_reserved":0,"allocated":3825664,"data_stored":2045219,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":87292357},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":9,"seq":38654706136,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":79400,"kb_used_data":3796,"kb_used_omap":1,"kb_used_meta":75582,"kb_avail":94292440,"statfs":{"total":96636764160,"available":96555458560,"internally_reserved":0,"allocated":3887104,"data_stored":2635606,"data_compressed":3546,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1595,"internal_metadata":77396421},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":9,"seq":38654706136,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":125160,"kb_used_data":3796,"kb_used_omap":1,"kb_used_meta":121342,"kb_avail":94246680,"statfs":{"total":96636764160,"available":96508600320,"internally_reserved":0,"allocated":3887104,"data_stored":2635606,"data_compressed":3546,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1595,"internal_metadata":124254661},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":3546,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":3546,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T09:11:46.981 DEBUG:teuthology.orchestra.run.smithi135:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2024-09-14T09:11:47.261 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T09:11:47.262 INFO:teuthology.orchestra.run.smithi135.stderr:dumped all 2024-09-14T09:11:47.274 INFO:teuthology.orchestra.run.smithi135.stdout:{"pg_ready":true,"pg_map":{"version":1633,"stamp":"2024-09-14T09:11:47.203579+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":81,"num_write_kb":1108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":293564,"kb_used_data":11328,"kb_used_omap":4,"kb_used_meta":282171,"kb_avail":282821956,"statfs":{"total":289910292480,"available":289609682944,"internally_reserved":0,"allocated":11599872,"data_stored":7316431,"data_compressed":7092,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":4785,"internal_metadata":288943439},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002250"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":934,"reported_epoch":465,"state":"active+clean","last_fresh":"2024-09-14T09:11:18.565126+0000","last_change":"2024-09-14T08:32:54.696724+0000","last_active":"2024-09-14T09:11:18.565126+0000","last_peered":"2024-09-14T09:11:18.565126+0000","last_clean":"2024-09-14T09:11:18.565126+0000","last_became_active":"2024-09-14T08:32:52.693987+0000","last_became_peered":"2024-09-14T08:32:52.693987+0000","last_unstale":"2024-09-14T09:11:18.565126+0000","last_undegraded":"2024-09-14T09:11:18.565126+0000","last_fullsized":"2024-09-14T09:11:18.565126+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:32:51.683516+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:32:51.683516+0000","last_clean_scrub_stamp":"2024-09-14T08:32:51.683516+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T17:10:29.729916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00111144,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":934,"reported_epoch":465,"state":"active+clean","last_fresh":"2024-09-14T09:11:18.565627+0000","last_change":"2024-09-14T08:32:54.696686+0000","last_active":"2024-09-14T09:11:18.565627+0000","last_peered":"2024-09-14T09:11:18.565627+0000","last_clean":"2024-09-14T09:11:18.565627+0000","last_became_active":"2024-09-14T08:32:52.691575+0000","last_became_peered":"2024-09-14T08:32:52.691575+0000","last_unstale":"2024-09-14T09:11:18.565627+0000","last_undegraded":"2024-09-14T09:11:18.565627+0000","last_fullsized":"2024-09-14T09:11:18.565627+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:32:51.683516+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:32:51.683516+0000","last_clean_scrub_stamp":"2024-09-14T08:32:51.683516+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T18:10:27.710040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001078809,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":935,"reported_epoch":465,"state":"active+clean","last_fresh":"2024-09-14T09:11:18.563618+0000","last_change":"2024-09-14T08:32:54.696401+0000","last_active":"2024-09-14T09:11:18.563618+0000","last_peered":"2024-09-14T09:11:18.563618+0000","last_clean":"2024-09-14T09:11:18.563618+0000","last_became_active":"2024-09-14T08:32:52.694100+0000","last_became_peered":"2024-09-14T08:32:52.694100+0000","last_unstale":"2024-09-14T09:11:18.563618+0000","last_undegraded":"2024-09-14T09:11:18.563618+0000","last_fullsized":"2024-09-14T09:11:18.563618+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:32:51.683516+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:32:51.683516+0000","last_clean_scrub_stamp":"2024-09-14T08:32:51.683516+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:48:32.158959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00082548099999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":935,"reported_epoch":465,"state":"active+clean","last_fresh":"2024-09-14T09:11:18.562068+0000","last_change":"2024-09-14T08:32:54.696398+0000","last_active":"2024-09-14T09:11:18.562068+0000","last_peered":"2024-09-14T09:11:18.562068+0000","last_clean":"2024-09-14T09:11:18.562068+0000","last_became_active":"2024-09-14T08:32:52.692903+0000","last_became_peered":"2024-09-14T08:32:52.692903+0000","last_unstale":"2024-09-14T09:11:18.562068+0000","last_undegraded":"2024-09-14T09:11:18.562068+0000","last_fullsized":"2024-09-14T09:11:18.562068+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:32:51.683516+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:32:51.683516+0000","last_clean_scrub_stamp":"2024-09-14T08:32:51.683516+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T20:22:32.946236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00085427300000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"16'2","reported_seq":937,"reported_epoch":465,"state":"active+clean","last_fresh":"2024-09-14T09:11:18.561824+0000","last_change":"2024-09-14T08:32:54.692785+0000","last_active":"2024-09-14T09:11:18.561824+0000","last_peered":"2024-09-14T09:11:18.561824+0000","last_clean":"2024-09-14T09:11:18.561824+0000","last_became_active":"2024-09-14T08:32:52.691649+0000","last_became_peered":"2024-09-14T08:32:52.691649+0000","last_unstale":"2024-09-14T09:11:18.561824+0000","last_undegraded":"2024-09-14T09:11:18.561824+0000","last_fullsized":"2024-09-14T09:11:18.561824+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:32:51.683516+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:32:51.683516+0000","last_clean_scrub_stamp":"2024-09-14T08:32:51.683516+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:27:21.636753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00094884699999999995,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":936,"reported_epoch":465,"state":"active+clean","last_fresh":"2024-09-14T09:11:18.549100+0000","last_change":"2024-09-14T08:32:54.689445+0000","last_active":"2024-09-14T09:11:18.549100+0000","last_peered":"2024-09-14T09:11:18.549100+0000","last_clean":"2024-09-14T09:11:18.549100+0000","last_became_active":"2024-09-14T08:32:52.693218+0000","last_became_peered":"2024-09-14T08:32:52.693218+0000","last_unstale":"2024-09-14T09:11:18.549100+0000","last_undegraded":"2024-09-14T09:11:18.549100+0000","last_fullsized":"2024-09-14T09:11:18.549100+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:32:51.683516+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:32:51.683516+0000","last_clean_scrub_stamp":"2024-09-14T08:32:51.683516+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T17:13:15.903145+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000454201,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":937,"reported_epoch":465,"state":"active+clean","last_fresh":"2024-09-14T09:11:18.549034+0000","last_change":"2024-09-14T08:32:54.689408+0000","last_active":"2024-09-14T09:11:18.549034+0000","last_peered":"2024-09-14T09:11:18.549034+0000","last_clean":"2024-09-14T09:11:18.549034+0000","last_became_active":"2024-09-14T08:32:52.693247+0000","last_became_peered":"2024-09-14T08:32:52.693247+0000","last_unstale":"2024-09-14T09:11:18.549034+0000","last_undegraded":"2024-09-14T09:11:18.549034+0000","last_fullsized":"2024-09-14T09:11:18.549034+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:32:51.683516+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:32:51.683516+0000","last_clean_scrub_stamp":"2024-09-14T08:32:51.683516+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:29:38.741547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00044293799999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"14'1","reported_seq":936,"reported_epoch":465,"state":"active+clean","last_fresh":"2024-09-14T09:11:18.562325+0000","last_change":"2024-09-14T08:32:54.697190+0000","last_active":"2024-09-14T09:11:18.562325+0000","last_peered":"2024-09-14T09:11:18.562325+0000","last_clean":"2024-09-14T09:11:18.562325+0000","last_became_active":"2024-09-14T08:32:52.691607+0000","last_became_peered":"2024-09-14T08:32:52.691607+0000","last_unstale":"2024-09-14T09:11:18.562325+0000","last_undegraded":"2024-09-14T09:11:18.562325+0000","last_fullsized":"2024-09-14T09:11:18.562325+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:32:51.683516+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:32:51.683516+0000","last_clean_scrub_stamp":"2024-09-14T08:32:51.683516+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T16:28:20.832608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0014408509999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"12'52","reported_seq":1002,"reported_epoch":465,"state":"active+clean","last_fresh":"2024-09-14T09:11:18.564234+0000","last_change":"2024-09-14T08:32:51.096141+0000","last_active":"2024-09-14T09:11:18.564234+0000","last_peered":"2024-09-14T09:11:18.564234+0000","last_clean":"2024-09-14T09:11:18.564234+0000","last_became_active":"2024-09-14T08:32:51.095164+0000","last_became_peered":"2024-09-14T08:32:51.095164+0000","last_unstale":"2024-09-14T09:11:18.564234+0000","last_undegraded":"2024-09-14T09:11:18.564234+0000","last_fullsized":"2024-09-14T09:11:18.564234+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:32:49.668980+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:32:49.668980+0000","last_clean_scrub_stamp":"2024-09-14T08:32:49.668980+0000","objects_scrubbed":0,"log_size":52,"log_dups_size":0,"ondisk_log_size":52,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T19:52:09.318667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":79,"num_write_kb":1106,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":79,"num_write_kb":1106,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":1180736,"data_compressed":7092,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":52,"ondisk_log_size":52,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654706137,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":89004,"kb_used_data":3736,"kb_used_omap":1,"kb_used_meta":85246,"kb_avail":94282836,"statfs":{"total":96636764160,"available":96545624064,"internally_reserved":0,"allocated":3825664,"data_stored":2045219,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":87292357},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":9,"seq":38654706136,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":79400,"kb_used_data":3796,"kb_used_omap":1,"kb_used_meta":75582,"kb_avail":94292440,"statfs":{"total":96636764160,"available":96555458560,"internally_reserved":0,"allocated":3887104,"data_stored":2635606,"data_compressed":3546,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1595,"internal_metadata":77396421},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":9,"seq":38654706136,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":125160,"kb_used_data":3796,"kb_used_omap":1,"kb_used_meta":121342,"kb_avail":94246680,"statfs":{"total":96636764160,"available":96508600320,"internally_reserved":0,"allocated":3887104,"data_stored":2635606,"data_compressed":3546,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1595,"internal_metadata":124254661},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":3546,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":3546,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T09:11:47.275 INFO:tasks.ceph.ceph_manager.cluster1:clean! 2024-09-14T09:11:47.275 DEBUG:teuthology.orchestra.run.smithi135:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2024-09-14T09:11:47.555 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T09:11:47.556 INFO:teuthology.orchestra.run.smithi135.stderr:dumped all 2024-09-14T09:11:47.568 INFO:teuthology.orchestra.run.smithi135.stdout:{"pg_ready":true,"pg_map":{"version":1633,"stamp":"2024-09-14T09:11:47.203579+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":81,"num_write_kb":1108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":293564,"kb_used_data":11328,"kb_used_omap":4,"kb_used_meta":282171,"kb_avail":282821956,"statfs":{"total":289910292480,"available":289609682944,"internally_reserved":0,"allocated":11599872,"data_stored":7316431,"data_compressed":7092,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":4785,"internal_metadata":288943439},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002250"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":934,"reported_epoch":465,"state":"active+clean","last_fresh":"2024-09-14T09:11:18.565126+0000","last_change":"2024-09-14T08:32:54.696724+0000","last_active":"2024-09-14T09:11:18.565126+0000","last_peered":"2024-09-14T09:11:18.565126+0000","last_clean":"2024-09-14T09:11:18.565126+0000","last_became_active":"2024-09-14T08:32:52.693987+0000","last_became_peered":"2024-09-14T08:32:52.693987+0000","last_unstale":"2024-09-14T09:11:18.565126+0000","last_undegraded":"2024-09-14T09:11:18.565126+0000","last_fullsized":"2024-09-14T09:11:18.565126+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:32:51.683516+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:32:51.683516+0000","last_clean_scrub_stamp":"2024-09-14T08:32:51.683516+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T17:10:29.729916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00111144,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":934,"reported_epoch":465,"state":"active+clean","last_fresh":"2024-09-14T09:11:18.565627+0000","last_change":"2024-09-14T08:32:54.696686+0000","last_active":"2024-09-14T09:11:18.565627+0000","last_peered":"2024-09-14T09:11:18.565627+0000","last_clean":"2024-09-14T09:11:18.565627+0000","last_became_active":"2024-09-14T08:32:52.691575+0000","last_became_peered":"2024-09-14T08:32:52.691575+0000","last_unstale":"2024-09-14T09:11:18.565627+0000","last_undegraded":"2024-09-14T09:11:18.565627+0000","last_fullsized":"2024-09-14T09:11:18.565627+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:32:51.683516+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:32:51.683516+0000","last_clean_scrub_stamp":"2024-09-14T08:32:51.683516+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T18:10:27.710040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001078809,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":935,"reported_epoch":465,"state":"active+clean","last_fresh":"2024-09-14T09:11:18.563618+0000","last_change":"2024-09-14T08:32:54.696401+0000","last_active":"2024-09-14T09:11:18.563618+0000","last_peered":"2024-09-14T09:11:18.563618+0000","last_clean":"2024-09-14T09:11:18.563618+0000","last_became_active":"2024-09-14T08:32:52.694100+0000","last_became_peered":"2024-09-14T08:32:52.694100+0000","last_unstale":"2024-09-14T09:11:18.563618+0000","last_undegraded":"2024-09-14T09:11:18.563618+0000","last_fullsized":"2024-09-14T09:11:18.563618+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:32:51.683516+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:32:51.683516+0000","last_clean_scrub_stamp":"2024-09-14T08:32:51.683516+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:48:32.158959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00082548099999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":935,"reported_epoch":465,"state":"active+clean","last_fresh":"2024-09-14T09:11:18.562068+0000","last_change":"2024-09-14T08:32:54.696398+0000","last_active":"2024-09-14T09:11:18.562068+0000","last_peered":"2024-09-14T09:11:18.562068+0000","last_clean":"2024-09-14T09:11:18.562068+0000","last_became_active":"2024-09-14T08:32:52.692903+0000","last_became_peered":"2024-09-14T08:32:52.692903+0000","last_unstale":"2024-09-14T09:11:18.562068+0000","last_undegraded":"2024-09-14T09:11:18.562068+0000","last_fullsized":"2024-09-14T09:11:18.562068+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:32:51.683516+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:32:51.683516+0000","last_clean_scrub_stamp":"2024-09-14T08:32:51.683516+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T20:22:32.946236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00085427300000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"16'2","reported_seq":937,"reported_epoch":465,"state":"active+clean","last_fresh":"2024-09-14T09:11:18.561824+0000","last_change":"2024-09-14T08:32:54.692785+0000","last_active":"2024-09-14T09:11:18.561824+0000","last_peered":"2024-09-14T09:11:18.561824+0000","last_clean":"2024-09-14T09:11:18.561824+0000","last_became_active":"2024-09-14T08:32:52.691649+0000","last_became_peered":"2024-09-14T08:32:52.691649+0000","last_unstale":"2024-09-14T09:11:18.561824+0000","last_undegraded":"2024-09-14T09:11:18.561824+0000","last_fullsized":"2024-09-14T09:11:18.561824+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:32:51.683516+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:32:51.683516+0000","last_clean_scrub_stamp":"2024-09-14T08:32:51.683516+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:27:21.636753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00094884699999999995,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":936,"reported_epoch":465,"state":"active+clean","last_fresh":"2024-09-14T09:11:18.549100+0000","last_change":"2024-09-14T08:32:54.689445+0000","last_active":"2024-09-14T09:11:18.549100+0000","last_peered":"2024-09-14T09:11:18.549100+0000","last_clean":"2024-09-14T09:11:18.549100+0000","last_became_active":"2024-09-14T08:32:52.693218+0000","last_became_peered":"2024-09-14T08:32:52.693218+0000","last_unstale":"2024-09-14T09:11:18.549100+0000","last_undegraded":"2024-09-14T09:11:18.549100+0000","last_fullsized":"2024-09-14T09:11:18.549100+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:32:51.683516+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:32:51.683516+0000","last_clean_scrub_stamp":"2024-09-14T08:32:51.683516+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T17:13:15.903145+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000454201,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":937,"reported_epoch":465,"state":"active+clean","last_fresh":"2024-09-14T09:11:18.549034+0000","last_change":"2024-09-14T08:32:54.689408+0000","last_active":"2024-09-14T09:11:18.549034+0000","last_peered":"2024-09-14T09:11:18.549034+0000","last_clean":"2024-09-14T09:11:18.549034+0000","last_became_active":"2024-09-14T08:32:52.693247+0000","last_became_peered":"2024-09-14T08:32:52.693247+0000","last_unstale":"2024-09-14T09:11:18.549034+0000","last_undegraded":"2024-09-14T09:11:18.549034+0000","last_fullsized":"2024-09-14T09:11:18.549034+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:32:51.683516+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:32:51.683516+0000","last_clean_scrub_stamp":"2024-09-14T08:32:51.683516+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:29:38.741547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00044293799999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"14'1","reported_seq":936,"reported_epoch":465,"state":"active+clean","last_fresh":"2024-09-14T09:11:18.562325+0000","last_change":"2024-09-14T08:32:54.697190+0000","last_active":"2024-09-14T09:11:18.562325+0000","last_peered":"2024-09-14T09:11:18.562325+0000","last_clean":"2024-09-14T09:11:18.562325+0000","last_became_active":"2024-09-14T08:32:52.691607+0000","last_became_peered":"2024-09-14T08:32:52.691607+0000","last_unstale":"2024-09-14T09:11:18.562325+0000","last_undegraded":"2024-09-14T09:11:18.562325+0000","last_fullsized":"2024-09-14T09:11:18.562325+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:32:51.683516+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:32:51.683516+0000","last_clean_scrub_stamp":"2024-09-14T08:32:51.683516+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T16:28:20.832608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0014408509999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"12'52","reported_seq":1002,"reported_epoch":465,"state":"active+clean","last_fresh":"2024-09-14T09:11:18.564234+0000","last_change":"2024-09-14T08:32:51.096141+0000","last_active":"2024-09-14T09:11:18.564234+0000","last_peered":"2024-09-14T09:11:18.564234+0000","last_clean":"2024-09-14T09:11:18.564234+0000","last_became_active":"2024-09-14T08:32:51.095164+0000","last_became_peered":"2024-09-14T08:32:51.095164+0000","last_unstale":"2024-09-14T09:11:18.564234+0000","last_undegraded":"2024-09-14T09:11:18.564234+0000","last_fullsized":"2024-09-14T09:11:18.564234+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:32:49.668980+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:32:49.668980+0000","last_clean_scrub_stamp":"2024-09-14T08:32:49.668980+0000","objects_scrubbed":0,"log_size":52,"log_dups_size":0,"ondisk_log_size":52,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T19:52:09.318667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":79,"num_write_kb":1106,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":79,"num_write_kb":1106,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":1180736,"data_compressed":7092,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":52,"ondisk_log_size":52,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654706137,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":89004,"kb_used_data":3736,"kb_used_omap":1,"kb_used_meta":85246,"kb_avail":94282836,"statfs":{"total":96636764160,"available":96545624064,"internally_reserved":0,"allocated":3825664,"data_stored":2045219,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":87292357},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":9,"seq":38654706136,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":79400,"kb_used_data":3796,"kb_used_omap":1,"kb_used_meta":75582,"kb_avail":94292440,"statfs":{"total":96636764160,"available":96555458560,"internally_reserved":0,"allocated":3887104,"data_stored":2635606,"data_compressed":3546,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1595,"internal_metadata":77396421},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":9,"seq":38654706136,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":125160,"kb_used_data":3796,"kb_used_omap":1,"kb_used_meta":121342,"kb_avail":94246680,"statfs":{"total":96636764160,"available":96508600320,"internally_reserved":0,"allocated":3887104,"data_stored":2635606,"data_compressed":3546,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1595,"internal_metadata":124254661},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":3546,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":3546,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T09:11:47.569 DEBUG:teuthology.orchestra.run.smithi135:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd dump --format=json 2024-09-14T09:11:47.853 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T09:11:47.853 INFO:teuthology.orchestra.run.smithi135.stdout:{"epoch":465,"fsid":"77c3c5ea-bdba-488a-b29d-8702bc143d62","created":"2024-09-14T08:32:36.219895+0000","modified":"2024-09-14T09:11:18.543762+0000","last_up_change":"2024-09-14T08:32:47.627053+0000","last_in_change":"2024-09-14T08:32:37.071724+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-14T08:32:48.810068+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-14T08:32:51.489753+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":16,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"72211b81-bcd2-4a02-a026-a5ee8497a3de","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":28,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6800","nonce":1099719938},{"type":"v1","addr":"172.21.15.135:6801","nonce":1099719938}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6802","nonce":1099719938},{"type":"v1","addr":"172.21.15.135:6803","nonce":1099719938}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6806","nonce":1099719938},{"type":"v1","addr":"172.21.15.135:6808","nonce":1099719938}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6804","nonce":1099719938},{"type":"v1","addr":"172.21.15.135:6805","nonce":1099719938}]},"public_addr":"172.21.15.135:6801/1099719938","cluster_addr":"172.21.15.135:6803/1099719938","heartbeat_back_addr":"172.21.15.135:6808/1099719938","heartbeat_front_addr":"172.21.15.135:6805/1099719938","state":["exists","up"]},{"osd":1,"uuid":"fed7a36b-99f5-45e7-8984-de5e09fba654","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":28,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6816","nonce":1622923706},{"type":"v1","addr":"172.21.15.135:6817","nonce":1622923706}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6818","nonce":1622923706},{"type":"v1","addr":"172.21.15.135:6819","nonce":1622923706}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6822","nonce":1622923706},{"type":"v1","addr":"172.21.15.135:6823","nonce":1622923706}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6820","nonce":1622923706},{"type":"v1","addr":"172.21.15.135:6821","nonce":1622923706}]},"public_addr":"172.21.15.135:6817/1622923706","cluster_addr":"172.21.15.135:6819/1622923706","heartbeat_back_addr":"172.21.15.135:6823/1622923706","heartbeat_front_addr":"172.21.15.135:6821/1622923706","state":["exists","up"]},{"osd":2,"uuid":"f39a5a76-8dc4-46f2-9e8d-c490677056f6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":28,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6807","nonce":1946554517},{"type":"v1","addr":"172.21.15.135:6809","nonce":1946554517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6810","nonce":1946554517},{"type":"v1","addr":"172.21.15.135:6811","nonce":1946554517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6814","nonce":1946554517},{"type":"v1","addr":"172.21.15.135:6815","nonce":1946554517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6812","nonce":1946554517},{"type":"v1","addr":"172.21.15.135:6813","nonce":1946554517}]},"public_addr":"172.21.15.135:6809/1946554517","cluster_addr":"172.21.15.135:6811/1946554517","heartbeat_back_addr":"172.21.15.135:6815/1946554517","heartbeat_front_addr":"172.21.15.135:6813/1946554517","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T08:32:43.433705+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T08:32:43.895674+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T08:32:43.286080+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-14T09:11:48.866 INFO:tasks.ceph:Scrubbing cluster1.osd.0 2024-09-14T09:11:48.867 DEBUG:teuthology.orchestra.run.smithi135:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.0 config set osd_debug_deep_scrub_sleep 0 2024-09-14T09:11:48.997 INFO:teuthology.orchestra.run.smithi135.stdout:{ 2024-09-14T09:11:48.997 INFO:teuthology.orchestra.run.smithi135.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) " 2024-09-14T09:11:48.997 INFO:teuthology.orchestra.run.smithi135.stdout:} 2024-09-14T09:11:49.008 DEBUG:teuthology.orchestra.run.smithi135:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd deep-scrub 0 2024-09-14T09:11:49.293 INFO:teuthology.orchestra.run.smithi135.stderr:instructed osd(s) 0 to deep-scrub 2024-09-14T09:11:49.307 INFO:tasks.ceph:Scrubbing cluster1.osd.1 2024-09-14T09:11:49.307 DEBUG:teuthology.orchestra.run.smithi135:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.1 config set osd_debug_deep_scrub_sleep 0 2024-09-14T09:11:49.429 INFO:teuthology.orchestra.run.smithi135.stdout:{ 2024-09-14T09:11:49.429 INFO:teuthology.orchestra.run.smithi135.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) " 2024-09-14T09:11:49.429 INFO:teuthology.orchestra.run.smithi135.stdout:} 2024-09-14T09:11:49.439 DEBUG:teuthology.orchestra.run.smithi135:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd deep-scrub 1 2024-09-14T09:11:49.726 INFO:teuthology.orchestra.run.smithi135.stderr:instructed osd(s) 1 to deep-scrub 2024-09-14T09:11:49.739 INFO:tasks.ceph:Scrubbing cluster1.osd.2 2024-09-14T09:11:49.739 DEBUG:teuthology.orchestra.run.smithi135:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.2 config set osd_debug_deep_scrub_sleep 0 2024-09-14T09:11:49.860 INFO:teuthology.orchestra.run.smithi135.stdout:{ 2024-09-14T09:11:49.860 INFO:teuthology.orchestra.run.smithi135.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) " 2024-09-14T09:11:49.860 INFO:teuthology.orchestra.run.smithi135.stdout:} 2024-09-14T09:11:49.871 DEBUG:teuthology.orchestra.run.smithi135:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd deep-scrub 2 2024-09-14T09:11:50.155 INFO:teuthology.orchestra.run.smithi135.stderr:instructed osd(s) 2 to deep-scrub 2024-09-14T09:11:50.168 DEBUG:teuthology.orchestra.run.smithi135:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2024-09-14T09:11:50.449 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T09:11:50.449 INFO:teuthology.orchestra.run.smithi135.stderr:dumped all 2024-09-14T09:11:50.461 INFO:teuthology.orchestra.run.smithi135.stdout:{"pg_ready":true,"pg_map":{"version":1634,"stamp":"2024-09-14T09:11:49.203893+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":81,"num_write_kb":1108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":293564,"kb_used_data":11328,"kb_used_omap":4,"kb_used_meta":282171,"kb_avail":282821956,"statfs":{"total":289910292480,"available":289609682944,"internally_reserved":0,"allocated":11599872,"data_stored":7316431,"data_compressed":7092,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":4785,"internal_metadata":288943439},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002200"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":934,"reported_epoch":465,"state":"active+clean","last_fresh":"2024-09-14T09:11:18.565126+0000","last_change":"2024-09-14T08:32:54.696724+0000","last_active":"2024-09-14T09:11:18.565126+0000","last_peered":"2024-09-14T09:11:18.565126+0000","last_clean":"2024-09-14T09:11:18.565126+0000","last_became_active":"2024-09-14T08:32:52.693987+0000","last_became_peered":"2024-09-14T08:32:52.693987+0000","last_unstale":"2024-09-14T09:11:18.565126+0000","last_undegraded":"2024-09-14T09:11:18.565126+0000","last_fullsized":"2024-09-14T09:11:18.565126+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:32:51.683516+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:32:51.683516+0000","last_clean_scrub_stamp":"2024-09-14T08:32:51.683516+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T17:10:29.729916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00111144,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":934,"reported_epoch":465,"state":"active+clean","last_fresh":"2024-09-14T09:11:18.565627+0000","last_change":"2024-09-14T08:32:54.696686+0000","last_active":"2024-09-14T09:11:18.565627+0000","last_peered":"2024-09-14T09:11:18.565627+0000","last_clean":"2024-09-14T09:11:18.565627+0000","last_became_active":"2024-09-14T08:32:52.691575+0000","last_became_peered":"2024-09-14T08:32:52.691575+0000","last_unstale":"2024-09-14T09:11:18.565627+0000","last_undegraded":"2024-09-14T09:11:18.565627+0000","last_fullsized":"2024-09-14T09:11:18.565627+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:32:51.683516+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:32:51.683516+0000","last_clean_scrub_stamp":"2024-09-14T08:32:51.683516+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T18:10:27.710040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001078809,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":935,"reported_epoch":465,"state":"active+clean","last_fresh":"2024-09-14T09:11:18.563618+0000","last_change":"2024-09-14T08:32:54.696401+0000","last_active":"2024-09-14T09:11:18.563618+0000","last_peered":"2024-09-14T09:11:18.563618+0000","last_clean":"2024-09-14T09:11:18.563618+0000","last_became_active":"2024-09-14T08:32:52.694100+0000","last_became_peered":"2024-09-14T08:32:52.694100+0000","last_unstale":"2024-09-14T09:11:18.563618+0000","last_undegraded":"2024-09-14T09:11:18.563618+0000","last_fullsized":"2024-09-14T09:11:18.563618+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:32:51.683516+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:32:51.683516+0000","last_clean_scrub_stamp":"2024-09-14T08:32:51.683516+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:48:32.158959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00082548099999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":935,"reported_epoch":465,"state":"active+clean","last_fresh":"2024-09-14T09:11:18.562068+0000","last_change":"2024-09-14T08:32:54.696398+0000","last_active":"2024-09-14T09:11:18.562068+0000","last_peered":"2024-09-14T09:11:18.562068+0000","last_clean":"2024-09-14T09:11:18.562068+0000","last_became_active":"2024-09-14T08:32:52.692903+0000","last_became_peered":"2024-09-14T08:32:52.692903+0000","last_unstale":"2024-09-14T09:11:18.562068+0000","last_undegraded":"2024-09-14T09:11:18.562068+0000","last_fullsized":"2024-09-14T09:11:18.562068+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:32:51.683516+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:32:51.683516+0000","last_clean_scrub_stamp":"2024-09-14T08:32:51.683516+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T20:22:32.946236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00085427300000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"16'2","reported_seq":937,"reported_epoch":465,"state":"active+clean","last_fresh":"2024-09-14T09:11:18.561824+0000","last_change":"2024-09-14T08:32:54.692785+0000","last_active":"2024-09-14T09:11:18.561824+0000","last_peered":"2024-09-14T09:11:18.561824+0000","last_clean":"2024-09-14T09:11:18.561824+0000","last_became_active":"2024-09-14T08:32:52.691649+0000","last_became_peered":"2024-09-14T08:32:52.691649+0000","last_unstale":"2024-09-14T09:11:18.561824+0000","last_undegraded":"2024-09-14T09:11:18.561824+0000","last_fullsized":"2024-09-14T09:11:18.561824+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:32:51.683516+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:32:51.683516+0000","last_clean_scrub_stamp":"2024-09-14T08:32:51.683516+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:27:21.636753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00094884699999999995,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":936,"reported_epoch":465,"state":"active+clean","last_fresh":"2024-09-14T09:11:18.549100+0000","last_change":"2024-09-14T08:32:54.689445+0000","last_active":"2024-09-14T09:11:18.549100+0000","last_peered":"2024-09-14T09:11:18.549100+0000","last_clean":"2024-09-14T09:11:18.549100+0000","last_became_active":"2024-09-14T08:32:52.693218+0000","last_became_peered":"2024-09-14T08:32:52.693218+0000","last_unstale":"2024-09-14T09:11:18.549100+0000","last_undegraded":"2024-09-14T09:11:18.549100+0000","last_fullsized":"2024-09-14T09:11:18.549100+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:32:51.683516+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:32:51.683516+0000","last_clean_scrub_stamp":"2024-09-14T08:32:51.683516+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T17:13:15.903145+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000454201,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":937,"reported_epoch":465,"state":"active+clean","last_fresh":"2024-09-14T09:11:18.549034+0000","last_change":"2024-09-14T08:32:54.689408+0000","last_active":"2024-09-14T09:11:18.549034+0000","last_peered":"2024-09-14T09:11:18.549034+0000","last_clean":"2024-09-14T09:11:18.549034+0000","last_became_active":"2024-09-14T08:32:52.693247+0000","last_became_peered":"2024-09-14T08:32:52.693247+0000","last_unstale":"2024-09-14T09:11:18.549034+0000","last_undegraded":"2024-09-14T09:11:18.549034+0000","last_fullsized":"2024-09-14T09:11:18.549034+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:32:51.683516+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:32:51.683516+0000","last_clean_scrub_stamp":"2024-09-14T08:32:51.683516+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:29:38.741547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00044293799999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"14'1","reported_seq":936,"reported_epoch":465,"state":"active+clean","last_fresh":"2024-09-14T09:11:18.562325+0000","last_change":"2024-09-14T08:32:54.697190+0000","last_active":"2024-09-14T09:11:18.562325+0000","last_peered":"2024-09-14T09:11:18.562325+0000","last_clean":"2024-09-14T09:11:18.562325+0000","last_became_active":"2024-09-14T08:32:52.691607+0000","last_became_peered":"2024-09-14T08:32:52.691607+0000","last_unstale":"2024-09-14T09:11:18.562325+0000","last_undegraded":"2024-09-14T09:11:18.562325+0000","last_fullsized":"2024-09-14T09:11:18.562325+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:32:51.683516+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:32:51.683516+0000","last_clean_scrub_stamp":"2024-09-14T08:32:51.683516+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T16:28:20.832608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0014408509999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"12'52","reported_seq":1002,"reported_epoch":465,"state":"active+clean","last_fresh":"2024-09-14T09:11:18.564234+0000","last_change":"2024-09-14T08:32:51.096141+0000","last_active":"2024-09-14T09:11:18.564234+0000","last_peered":"2024-09-14T09:11:18.564234+0000","last_clean":"2024-09-14T09:11:18.564234+0000","last_became_active":"2024-09-14T08:32:51.095164+0000","last_became_peered":"2024-09-14T08:32:51.095164+0000","last_unstale":"2024-09-14T09:11:18.564234+0000","last_undegraded":"2024-09-14T09:11:18.564234+0000","last_fullsized":"2024-09-14T09:11:18.564234+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T08:32:49.668980+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T08:32:49.668980+0000","last_clean_scrub_stamp":"2024-09-14T08:32:49.668980+0000","objects_scrubbed":0,"log_size":52,"log_dups_size":0,"ondisk_log_size":52,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T19:52:09.318667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":79,"num_write_kb":1106,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":79,"num_write_kb":1106,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":1180736,"data_compressed":7092,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":52,"ondisk_log_size":52,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654706138,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":89004,"kb_used_data":3736,"kb_used_omap":1,"kb_used_meta":85246,"kb_avail":94282836,"statfs":{"total":96636764160,"available":96545624064,"internally_reserved":0,"allocated":3825664,"data_stored":2045219,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":87292357},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":9,"seq":38654706137,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":79400,"kb_used_data":3796,"kb_used_omap":1,"kb_used_meta":75582,"kb_avail":94292440,"statfs":{"total":96636764160,"available":96555458560,"internally_reserved":0,"allocated":3887104,"data_stored":2635606,"data_compressed":3546,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1595,"internal_metadata":77396421},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":9,"seq":38654706137,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":125160,"kb_used_data":3796,"kb_used_omap":1,"kb_used_meta":121342,"kb_avail":94246680,"statfs":{"total":96636764160,"available":96508600320,"internally_reserved":0,"allocated":3887104,"data_stored":2635606,"data_compressed":3546,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1595,"internal_metadata":124254661},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":3546,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":3546,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T09:11:50.462 INFO:tasks.ceph:pgid 2.7 last_scrub_stamp 2024-09-14T08:32:51.683516+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=14, tm_hour=8, tm_min=32, tm_sec=51, tm_wday=5, tm_yday=258, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=14, tm_hour=9, tm_min=11, tm_sec=47, tm_wday=5, tm_yday=258, tm_isdst=0) 2024-09-14T09:11:50.462 INFO:tasks.ceph:pgid 2.6 last_scrub_stamp 2024-09-14T08:32:51.683516+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=14, tm_hour=8, tm_min=32, tm_sec=51, tm_wday=5, tm_yday=258, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=14, tm_hour=9, tm_min=11, tm_sec=47, tm_wday=5, tm_yday=258, tm_isdst=0) 2024-09-14T09:11:50.463 INFO:tasks.ceph:pgid 2.5 last_scrub_stamp 2024-09-14T08:32:51.683516+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=14, tm_hour=8, tm_min=32, tm_sec=51, tm_wday=5, tm_yday=258, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=14, tm_hour=9, tm_min=11, tm_sec=47, tm_wday=5, tm_yday=258, tm_isdst=0) 2024-09-14T09:11:50.463 INFO:tasks.ceph:pgid 2.4 last_scrub_stamp 2024-09-14T08:32:51.683516+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=14, tm_hour=8, tm_min=32, tm_sec=51, tm_wday=5, tm_yday=258, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=14, tm_hour=9, tm_min=11, tm_sec=47, tm_wday=5, tm_yday=258, tm_isdst=0) 2024-09-14T09:11:50.463 INFO:tasks.ceph:pgid 2.2 last_scrub_stamp 2024-09-14T08:32:51.683516+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=14, tm_hour=8, tm_min=32, tm_sec=51, tm_wday=5, tm_yday=258, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=14, tm_hour=9, tm_min=11, tm_sec=47, tm_wday=5, tm_yday=258, tm_isdst=0) 2024-09-14T09:11:50.463 INFO:tasks.ceph:pgid 2.1 last_scrub_stamp 2024-09-14T08:32:51.683516+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=14, tm_hour=8, tm_min=32, tm_sec=51, tm_wday=5, tm_yday=258, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=14, tm_hour=9, tm_min=11, tm_sec=47, tm_wday=5, tm_yday=258, tm_isdst=0) 2024-09-14T09:11:50.463 INFO:tasks.ceph:pgid 2.0 last_scrub_stamp 2024-09-14T08:32:51.683516+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=14, tm_hour=8, tm_min=32, tm_sec=51, tm_wday=5, tm_yday=258, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=14, tm_hour=9, tm_min=11, tm_sec=47, tm_wday=5, tm_yday=258, tm_isdst=0) 2024-09-14T09:11:50.463 INFO:tasks.ceph:pgid 2.3 last_scrub_stamp 2024-09-14T08:32:51.683516+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=14, tm_hour=8, tm_min=32, tm_sec=51, tm_wday=5, tm_yday=258, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=14, tm_hour=9, tm_min=11, tm_sec=47, tm_wday=5, tm_yday=258, tm_isdst=0) 2024-09-14T09:11:50.463 INFO:tasks.ceph:pgid 1.0 last_scrub_stamp 2024-09-14T08:32:49.668980+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=14, tm_hour=8, tm_min=32, tm_sec=49, tm_wday=5, tm_yday=258, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=14, tm_hour=9, tm_min=11, tm_sec=47, tm_wday=5, tm_yday=258, tm_isdst=0) 2024-09-14T09:11:50.463 INFO:tasks.ceph:Still waiting for all pgs to be scrubbed. 2024-09-14T09:12:10.465 DEBUG:teuthology.orchestra.run.smithi135:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2024-09-14T09:12:10.757 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-09-14T09:12:10.757 INFO:teuthology.orchestra.run.smithi135.stderr:dumped all 2024-09-14T09:12:10.769 INFO:teuthology.orchestra.run.smithi135.stdout:{"pg_ready":true,"pg_map":{"version":1644,"stamp":"2024-09-14T09:12:09.208049+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":81,"num_write_kb":1108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":59,"ondisk_log_size":59,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":293552,"kb_used_data":11328,"kb_used_omap":4,"kb_used_meta":282171,"kb_avail":282821968,"statfs":{"total":289910292480,"available":289609695232,"internally_reserved":0,"allocated":11599872,"data_stored":7316431,"data_compressed":7092,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":4785,"internal_metadata":288943439},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002573"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":943,"reported_epoch":465,"state":"active+clean","last_fresh":"2024-09-14T09:11:55.358289+0000","last_change":"2024-09-14T09:11:55.358289+0000","last_active":"2024-09-14T09:11:55.358289+0000","last_peered":"2024-09-14T09:11:55.358289+0000","last_clean":"2024-09-14T09:11:55.358289+0000","last_became_active":"2024-09-14T08:32:52.693987+0000","last_became_peered":"2024-09-14T08:32:52.693987+0000","last_unstale":"2024-09-14T09:11:55.358289+0000","last_undegraded":"2024-09-14T09:11:55.358289+0000","last_fullsized":"2024-09-14T09:11:55.358289+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T09:11:55.358198+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T09:11:55.358198+0000","last_clean_scrub_stamp":"2024-09-14T09:11:55.358198+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:14:50.432878+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0.00111144,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":943,"reported_epoch":465,"state":"active+clean","last_fresh":"2024-09-14T09:11:54.322467+0000","last_change":"2024-09-14T09:11:54.322467+0000","last_active":"2024-09-14T09:11:54.322467+0000","last_peered":"2024-09-14T09:11:54.322467+0000","last_clean":"2024-09-14T09:11:54.322467+0000","last_became_active":"2024-09-14T08:32:52.691575+0000","last_became_peered":"2024-09-14T08:32:52.691575+0000","last_unstale":"2024-09-14T09:11:54.322467+0000","last_undegraded":"2024-09-14T09:11:54.322467+0000","last_fullsized":"2024-09-14T09:11:54.322467+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T09:11:54.322367+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T09:11:54.322367+0000","last_clean_scrub_stamp":"2024-09-14T09:11:54.322367+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:03:05.224026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001078809,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":943,"reported_epoch":465,"state":"active+clean","last_fresh":"2024-09-14T09:11:53.364451+0000","last_change":"2024-09-14T09:11:53.364451+0000","last_active":"2024-09-14T09:11:53.364451+0000","last_peered":"2024-09-14T09:11:53.364451+0000","last_clean":"2024-09-14T09:11:53.364451+0000","last_became_active":"2024-09-14T08:32:52.694100+0000","last_became_peered":"2024-09-14T08:32:52.694100+0000","last_unstale":"2024-09-14T09:11:53.364451+0000","last_undegraded":"2024-09-14T09:11:53.364451+0000","last_fullsized":"2024-09-14T09:11:53.364451+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T09:11:53.364355+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T09:11:53.364355+0000","last_clean_scrub_stamp":"2024-09-14T09:11:53.364355+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T18:05:01.491267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00082548099999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":943,"reported_epoch":465,"state":"active+clean","last_fresh":"2024-09-14T09:11:52.391716+0000","last_change":"2024-09-14T09:11:52.391716+0000","last_active":"2024-09-14T09:11:52.391716+0000","last_peered":"2024-09-14T09:11:52.391716+0000","last_clean":"2024-09-14T09:11:52.391716+0000","last_became_active":"2024-09-14T08:32:52.692903+0000","last_became_peered":"2024-09-14T08:32:52.692903+0000","last_unstale":"2024-09-14T09:11:52.391716+0000","last_undegraded":"2024-09-14T09:11:52.391716+0000","last_fullsized":"2024-09-14T09:11:52.391716+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T09:11:52.391634+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T09:11:52.391634+0000","last_clean_scrub_stamp":"2024-09-14T09:11:52.391634+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T20:57:22.300799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00085427300000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"465'3","reported_seq":946,"reported_epoch":465,"state":"active+clean","last_fresh":"2024-09-14T09:11:49.614380+0000","last_change":"2024-09-14T09:11:49.614380+0000","last_active":"2024-09-14T09:11:49.614380+0000","last_peered":"2024-09-14T09:11:49.614380+0000","last_clean":"2024-09-14T09:11:49.614380+0000","last_became_active":"2024-09-14T08:32:52.691649+0000","last_became_peered":"2024-09-14T08:32:52.691649+0000","last_unstale":"2024-09-14T09:11:49.614380+0000","last_undegraded":"2024-09-14T09:11:49.614380+0000","last_fullsized":"2024-09-14T09:11:49.614380+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"465'3","last_scrub_stamp":"2024-09-14T09:11:49.614261+0000","last_deep_scrub":"465'3","last_deep_scrub_stamp":"2024-09-14T09:11:49.614261+0000","last_clean_scrub_stamp":"2024-09-14T09:11:49.614261+0000","objects_scrubbed":1,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:25:14.451893+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0.00094884699999999995,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":944,"reported_epoch":465,"state":"active+clean","last_fresh":"2024-09-14T09:11:51.950637+0000","last_change":"2024-09-14T09:11:51.950637+0000","last_active":"2024-09-14T09:11:51.950637+0000","last_peered":"2024-09-14T09:11:51.950637+0000","last_clean":"2024-09-14T09:11:51.950637+0000","last_became_active":"2024-09-14T08:32:52.693218+0000","last_became_peered":"2024-09-14T08:32:52.693218+0000","last_unstale":"2024-09-14T09:11:51.950637+0000","last_undegraded":"2024-09-14T09:11:51.950637+0000","last_fullsized":"2024-09-14T09:11:51.950637+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T09:11:51.950552+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T09:11:51.950552+0000","last_clean_scrub_stamp":"2024-09-14T09:11:51.950552+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T16:26:30.630958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000454201,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":945,"reported_epoch":465,"state":"active+clean","last_fresh":"2024-09-14T09:11:50.990461+0000","last_change":"2024-09-14T09:11:50.990461+0000","last_active":"2024-09-14T09:11:50.990461+0000","last_peered":"2024-09-14T09:11:50.990461+0000","last_clean":"2024-09-14T09:11:50.990461+0000","last_became_active":"2024-09-14T08:32:52.693247+0000","last_became_peered":"2024-09-14T08:32:52.693247+0000","last_unstale":"2024-09-14T09:11:50.990461+0000","last_undegraded":"2024-09-14T09:11:50.990461+0000","last_fullsized":"2024-09-14T09:11:50.990461+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T09:11:50.990383+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T09:11:50.990383+0000","last_clean_scrub_stamp":"2024-09-14T09:11:50.990383+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T16:05:26.620046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00044293799999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"465'2","reported_seq":945,"reported_epoch":465,"state":"active+clean","last_fresh":"2024-09-14T09:11:51.433891+0000","last_change":"2024-09-14T09:11:51.433891+0000","last_active":"2024-09-14T09:11:51.433891+0000","last_peered":"2024-09-14T09:11:51.433891+0000","last_clean":"2024-09-14T09:11:51.433891+0000","last_became_active":"2024-09-14T08:32:52.691607+0000","last_became_peered":"2024-09-14T08:32:52.691607+0000","last_unstale":"2024-09-14T09:11:51.433891+0000","last_undegraded":"2024-09-14T09:11:51.433891+0000","last_fullsized":"2024-09-14T09:11:51.433891+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"465'2","last_scrub_stamp":"2024-09-14T09:11:51.433784+0000","last_deep_scrub":"465'2","last_deep_scrub_stamp":"2024-09-14T09:11:51.433784+0000","last_clean_scrub_stamp":"2024-09-14T09:11:51.433784+0000","objects_scrubbed":1,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:06:59.988047+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0.0014408509999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"465'54","reported_seq":1012,"reported_epoch":465,"state":"active+clean","last_fresh":"2024-09-14T09:11:50.442853+0000","last_change":"2024-09-14T09:11:50.442853+0000","last_active":"2024-09-14T09:11:50.442853+0000","last_peered":"2024-09-14T09:11:50.442853+0000","last_clean":"2024-09-14T09:11:50.442853+0000","last_became_active":"2024-09-14T08:32:51.095164+0000","last_became_peered":"2024-09-14T08:32:51.095164+0000","last_unstale":"2024-09-14T09:11:50.442853+0000","last_undegraded":"2024-09-14T09:11:50.442853+0000","last_fullsized":"2024-09-14T09:11:50.442853+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"465'54","last_scrub_stamp":"2024-09-14T09:11:50.442758+0000","last_deep_scrub":"465'54","last_deep_scrub_stamp":"2024-09-14T09:11:50.442758+0000","last_clean_scrub_stamp":"2024-09-14T09:11:50.442758+0000","objects_scrubbed":2,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T21:00:43.353034+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":79,"num_write_kb":1106,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":5,"ondisk_log_size":5,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":79,"num_write_kb":1106,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":1180736,"data_compressed":7092,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":54,"ondisk_log_size":54,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654706142,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":89000,"kb_used_data":3736,"kb_used_omap":1,"kb_used_meta":85246,"kb_avail":94282840,"statfs":{"total":96636764160,"available":96545628160,"internally_reserved":0,"allocated":3825664,"data_stored":2045219,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":87292357},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":9,"seq":38654706141,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":79396,"kb_used_data":3796,"kb_used_omap":1,"kb_used_meta":75582,"kb_avail":94292444,"statfs":{"total":96636764160,"available":96555462656,"internally_reserved":0,"allocated":3887104,"data_stored":2635606,"data_compressed":3546,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1595,"internal_metadata":77396421},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":9,"seq":38654706141,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":125156,"kb_used_data":3796,"kb_used_omap":1,"kb_used_meta":121342,"kb_avail":94246684,"statfs":{"total":96636764160,"available":96508604416,"internally_reserved":0,"allocated":3887104,"data_stored":2635606,"data_compressed":3546,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1595,"internal_metadata":124254661},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":3546,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":3546,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T09:12:10.771 DEBUG:teuthology.orchestra.run.smithi135:> sudo ceph --cluster cluster1 config set global mon_health_to_clog false 2024-09-14T09:12:11.073 INFO:teuthology.misc:Shutting down mds daemons... 2024-09-14T09:12:11.073 INFO:teuthology.misc:Shutting down osd daemons... 2024-09-14T09:12:11.073 DEBUG:tasks.ceph.cluster1.osd.0:waiting for process to exit 2024-09-14T09:12:11.073 INFO:teuthology.orchestra.run:waiting for 300 2024-09-14T09:12:11.121 INFO:tasks.ceph.cluster1.osd.0:Stopped 2024-09-14T09:12:11.121 DEBUG:tasks.ceph.cluster1.osd.1:waiting for process to exit 2024-09-14T09:12:11.121 INFO:teuthology.orchestra.run:waiting for 300 2024-09-14T09:12:11.159 INFO:tasks.ceph.cluster1.osd.1:Stopped 2024-09-14T09:12:11.159 DEBUG:tasks.ceph.cluster1.osd.2:waiting for process to exit 2024-09-14T09:12:11.159 INFO:teuthology.orchestra.run:waiting for 300 2024-09-14T09:12:11.232 INFO:tasks.ceph.cluster1.osd.2:Stopped 2024-09-14T09:12:11.232 INFO:teuthology.misc:Shutting down mgr daemons... 2024-09-14T09:12:11.232 DEBUG:tasks.ceph.cluster1.mgr.x:waiting for process to exit 2024-09-14T09:12:11.233 INFO:teuthology.orchestra.run:waiting for 300 2024-09-14T09:12:11.300 INFO:tasks.ceph.cluster1.mgr.x:Stopped 2024-09-14T09:12:11.300 INFO:teuthology.misc:Shutting down mon daemons... 2024-09-14T09:12:11.301 DEBUG:tasks.ceph.cluster1.mon.a:waiting for process to exit 2024-09-14T09:12:11.301 INFO:teuthology.orchestra.run:waiting for 300 2024-09-14T09:12:11.353 INFO:tasks.ceph.cluster1.mon.a:Stopped 2024-09-14T09:12:11.353 INFO:tasks.ceph:Checking cluster log for badness... 2024-09-14T09:12:11.353 DEBUG:teuthology.orchestra.run.smithi135:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/cluster1.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v '\(OSD_SLOW_PING_TIME' | head -n 1 2024-09-14T09:12:11.422 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster1-0 on ubuntu@smithi135.front.sepia.ceph.com 2024-09-14T09:12:11.422 DEBUG:teuthology.orchestra.run.smithi135:> sync && sudo umount -f /var/lib/ceph/osd/cluster1-0 2024-09-14T09:12:11.661 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster1-1 on ubuntu@smithi135.front.sepia.ceph.com 2024-09-14T09:12:11.662 DEBUG:teuthology.orchestra.run.smithi135:> sync && sudo umount -f /var/lib/ceph/osd/cluster1-1 2024-09-14T09:12:11.784 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster1-2 on ubuntu@smithi135.front.sepia.ceph.com 2024-09-14T09:12:11.784 DEBUG:teuthology.orchestra.run.smithi135:> sync && sudo umount -f /var/lib/ceph/osd/cluster1-2 2024-09-14T09:12:11.850 INFO:tasks.ceph:Archiving mon data... 2024-09-14T09:12:11.850 DEBUG:teuthology.misc:Transferring archived files from smithi135:/var/lib/ceph/mon/cluster1-a to /home/teuthworker/archive/teuthology-2024-09-10_20:16:17-rbd-main-distro-default-smithi/7900492/data/cluster1.mon.a.tgz 2024-09-14T09:12:11.850 DEBUG:teuthology.orchestra.run.smithi135:> mktemp 2024-09-14T09:12:11.869 INFO:teuthology.orchestra.run.smithi135.stdout:/tmp/tmp.1DHsErIqAz 2024-09-14T09:12:11.869 DEBUG:teuthology.orchestra.run.smithi135:> sudo tar cz -f - -C /var/lib/ceph/mon/cluster1-a -- . > /tmp/tmp.1DHsErIqAz 2024-09-14T09:12:12.053 DEBUG:teuthology.orchestra.run.smithi135:> sudo chmod 0666 /tmp/tmp.1DHsErIqAz 2024-09-14T09:12:12.136 DEBUG:teuthology.orchestra.remote:smithi135:/tmp/tmp.1DHsErIqAz is 402KB 2024-09-14T09:12:12.231 DEBUG:teuthology.orchestra.run.smithi135:> rm -fr /tmp/tmp.1DHsErIqAz 2024-09-14T09:12:12.245 INFO:tasks.ceph:Cleaning ceph cluster... 2024-09-14T09:12:12.246 DEBUG:teuthology.orchestra.run.smithi135:> sudo rm -rf -- /etc/ceph/cluster1.conf /etc/ceph/cluster1.keyring /home/ubuntu/cephtest/cluster1.data /home/ubuntu/cephtest/cluster1.monmap /home/ubuntu/cephtest/../*.pid 2024-09-14T09:12:12.288 DEBUG:teuthology.orchestra.run.smithi164:> sudo rm -rf -- /etc/ceph/cluster1.conf /etc/ceph/cluster1.keyring /home/ubuntu/cephtest/cluster1.data /home/ubuntu/cephtest/cluster1.monmap /home/ubuntu/cephtest/../*.pid 2024-09-14T09:12:12.367 INFO:teuthology.util.scanner:summary_data or yaml_file is empty! 2024-09-14T09:12:12.385 INFO:teuthology.util.scanner:summary_data or yaml_file is empty! 2024-09-14T09:12:12.386 INFO:tasks.ceph:Archiving crash dumps... 2024-09-14T09:12:12.387 DEBUG:teuthology.misc:Transferring archived files from smithi135:/var/lib/ceph/crash to /home/teuthworker/archive/teuthology-2024-09-10_20:16:17-rbd-main-distro-default-smithi/7900492/remote/smithi135/crash 2024-09-14T09:12:12.389 DEBUG:teuthology.orchestra.run.smithi135:> sudo tar c -f - -C /var/lib/ceph/crash -- . 2024-09-14T09:12:12.432 DEBUG:teuthology.misc:Transferring archived files from smithi164:/var/lib/ceph/crash to /home/teuthworker/archive/teuthology-2024-09-10_20:16:17-rbd-main-distro-default-smithi/7900492/remote/smithi164/crash 2024-09-14T09:12:12.433 DEBUG:teuthology.orchestra.run.smithi164:> sudo tar c -f - -C /var/lib/ceph/crash -- . 2024-09-14T09:12:12.462 INFO:tasks.ceph:Compressing logs... 2024-09-14T09:12:12.462 DEBUG:teuthology.orchestra.run.smithi135:> time sudo find /var/log/ceph -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-09-14T09:12:12.476 DEBUG:teuthology.orchestra.run.smithi164:> time sudo find /var/log/ceph -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-09-14T09:12:12.519 INFO:teuthology.orchestra.run.smithi135.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.37378.log 2024-09-14T09:12:12.519 INFO:teuthology.orchestra.run.smithi135.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.37042.log 2024-09-14T09:12:12.519 INFO:teuthology.orchestra.run.smithi135.stderr:gzip -5 --verbose/var/log/ceph/cluster1-client.admin.37378.log: -- /var/log/ceph/cluster1-client.admin.35924.log 2024-09-14T09:12:12.519 INFO:teuthology.orchestra.run.smithi135.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.37378.log.gz 2024-09-14T09:12:12.519 INFO:teuthology.orchestra.run.smithi135.stderr:/var/log/ceph/cluster1-client.admin.37042.log: gzip -5 --verbose -- /var/log/ceph/cluster1-mon.a.log 2024-09-14T09:12:12.519 INFO:teuthology.orchestra.run.smithi135.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.37042.log.gz 2024-09-14T09:12:12.519 INFO:teuthology.orchestra.run.smithi135.stderr:/var/log/ceph/cluster1-client.admin.35924.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.37462.log 2024-09-14T09:12:12.519 INFO:teuthology.orchestra.run.smithi135.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.35924.log.gz 2024-09-14T09:12:12.519 INFO:teuthology.orchestra.run.smithi135.stderr:/var/log/ceph/cluster1-mon.a.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.37168.log 2024-09-14T09:12:12.519 INFO:teuthology.orchestra.run.smithi135.stderr:/var/log/ceph/cluster1-client.admin.37462.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.37084.log 2024-09-14T09:12:12.519 INFO:teuthology.orchestra.run.smithi135.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.37462.log.gz 2024-09-14T09:12:12.520 INFO:teuthology.orchestra.run.smithi135.stderr:/var/log/ceph/cluster1-client.admin.37168.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.35624.log 2024-09-14T09:12:12.520 INFO:teuthology.orchestra.run.smithi135.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.37168.log.gz 2024-09-14T09:12:12.520 INFO:teuthology.orchestra.run.smithi135.stderr:/var/log/ceph/cluster1-client.admin.37084.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.32003.log 2024-09-14T09:12:12.520 INFO:teuthology.orchestra.run.smithi135.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.37084.log.gz 2024-09-14T09:12:12.520 INFO:teuthology.orchestra.run.smithi135.stderr:/var/log/ceph/cluster1-client.admin.35624.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.35624.log.gzgzip 2024-09-14T09:12:12.520 INFO:teuthology.orchestra.run.smithi135.stderr: -5 --verbose -- /var/log/ceph/cluster1-osd.1.log 2024-09-14T09:12:12.520 INFO:teuthology.orchestra.run.smithi135.stderr:/var/log/ceph/cluster1-client.admin.32003.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.35837.log 2024-09-14T09:12:12.520 INFO:teuthology.orchestra.run.smithi135.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.32003.log.gz 2024-09-14T09:12:12.520 INFO:teuthology.orchestra.run.smithi135.stderr:/var/log/ceph/cluster1-osd.1.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.35711.log 2024-09-14T09:12:12.520 INFO:teuthology.orchestra.run.smithi135.stderr:/var/log/ceph/cluster1-client.admin.35837.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.37210.log 2024-09-14T09:12:12.520 INFO:teuthology.orchestra.run.smithi135.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.35837.log.gz 2024-09-14T09:12:12.520 INFO:teuthology.orchestra.run.smithi135.stderr:/var/log/ceph/cluster1-client.admin.35711.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.35711.log.gz 2024-09-14T09:12:12.520 INFO:teuthology.orchestra.run.smithi135.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.35497.log 2024-09-14T09:12:12.520 INFO:teuthology.orchestra.run.smithi135.stderr:/var/log/ceph/cluster1-client.admin.37210.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.35498.log 2024-09-14T09:12:12.521 INFO:teuthology.orchestra.run.smithi135.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.37210.log.gz 2024-09-14T09:12:12.521 INFO:teuthology.orchestra.run.smithi135.stderr:/var/log/ceph/cluster1-client.admin.35497.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.35497.log.gz 2024-09-14T09:12:12.521 INFO:teuthology.orchestra.run.smithi135.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.37252.log 2024-09-14T09:12:12.521 INFO:teuthology.orchestra.run.smithi135.stderr:/var/log/ceph/cluster1-client.admin.35498.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.35190.log 2024-09-14T09:12:12.521 INFO:teuthology.orchestra.run.smithi135.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.35498.log.gz 2024-09-14T09:12:12.521 INFO:teuthology.orchestra.run.smithi135.stderr:/var/log/ceph/cluster1-client.admin.37252.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.37252.log.gz 2024-09-14T09:12:12.521 INFO:teuthology.orchestra.run.smithi135.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.34980.log 2024-09-14T09:12:12.521 INFO:teuthology.orchestra.run.smithi135.stderr:/var/log/ceph/cluster1-client.admin.35190.log: gzip 0.0% -5 -- replaced with /var/log/ceph/cluster1-client.admin.35190.log.gz --verbose 2024-09-14T09:12:12.521 INFO:teuthology.orchestra.run.smithi135.stderr: -- /var/log/ceph/cluster1-client.admin.37336.log 2024-09-14T09:12:12.521 INFO:teuthology.orchestra.run.smithi135.stderr:/var/log/ceph/cluster1-client.admin.34980.log: gzip -5 --verbose -- /var/log/ceph/cluster1-osd.2.log 2024-09-14T09:12:12.521 INFO:teuthology.orchestra.run.smithi135.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.34980.log.gz 2024-09-14T09:12:12.521 INFO:teuthology.orchestra.run.smithi135.stderr:/var/log/ceph/cluster1-client.admin.37336.log: gzip -5 --verbose -- /var/log/ceph/cluster2-mgr.x.log 2024-09-14T09:12:12.521 INFO:teuthology.orchestra.run.smithi135.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.37336.log.gz 2024-09-14T09:12:12.522 INFO:teuthology.orchestra.run.smithi135.stderr:/var/log/ceph/cluster1-osd.2.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.35375.log 2024-09-14T09:12:12.522 INFO:teuthology.orchestra.run.smithi135.stderr:/var/log/ceph/cluster2-mgr.x.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.37420.log 2024-09-14T09:12:12.522 INFO:teuthology.orchestra.run.smithi135.stderr:/var/log/ceph/cluster1-client.admin.35375.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.35585.log 2024-09-14T09:12:12.522 INFO:teuthology.orchestra.run.smithi135.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.35375.log.gz 2024-09-14T09:12:12.522 INFO:teuthology.orchestra.run.smithi135.stderr:/var/log/ceph/cluster1-client.admin.37420.log: gzip -5 --verbose -- /var/log/ceph/cluster1-mgr.x.log 2024-09-14T09:12:12.522 INFO:teuthology.orchestra.run.smithi135.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.37420.log.gz 2024-09-14T09:12:12.522 INFO:teuthology.orchestra.run.smithi135.stderr:/var/log/ceph/cluster1-client.admin.35585.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.37126.log 2024-09-14T09:12:12.522 INFO:teuthology.orchestra.run.smithi135.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.35585.log.gz 2024-09-14T09:12:12.522 INFO:teuthology.orchestra.run.smithi135.stderr:/var/log/ceph/cluster1-mgr.x.log: gzip -5 --verbose -- /var/log/ceph/ceph.tmp-client.admin.27510.log 2024-09-14T09:12:12.522 INFO:teuthology.orchestra.run.smithi135.stderr:/var/log/ceph/cluster1-client.admin.37126.log: gzip 0.0% -5 --verbose -- /var/log/ceph/cluster1-client.admin.32241.log 2024-09-14T09:12:12.522 INFO:teuthology.orchestra.run.smithi135.stderr: -- replaced with /var/log/ceph/cluster1-client.admin.37126.log.gz 2024-09-14T09:12:12.522 INFO:teuthology.orchestra.run.smithi135.stderr:/var/log/ceph/ceph.tmp-client.admin.27510.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.32405.log 2024-09-14T09:12:12.522 INFO:teuthology.orchestra.run.smithi135.stderr: 0.0% -- replaced with /var/log/ceph/ceph.tmp-client.admin.27510.log.gz 2024-09-14T09:12:12.522 INFO:teuthology.orchestra.run.smithi135.stderr:gzip -5/var/log/ceph/cluster1-client.admin.32241.log: --verbose -- /var/log/ceph/cluster1-client.admin.35499.log 2024-09-14T09:12:12.522 INFO:teuthology.orchestra.run.smithi135.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.32241.log.gz 2024-09-14T09:12:12.522 INFO:teuthology.orchestra.run.smithi135.stderr:/var/log/ceph/cluster1-client.admin.32405.log: gzip -5 --verbose 26.9% -- replaced with /var/log/ceph/cluster1-client.admin.32405.log.gz 2024-09-14T09:12:12.523 INFO:teuthology.orchestra.run.smithi135.stderr: -- /var/log/ceph/cluster1-client.admin.37505.log 2024-09-14T09:12:12.523 INFO:teuthology.orchestra.run.smithi135.stderr:/var/log/ceph/cluster1-client.admin.35499.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.35499.log.gz 2024-09-14T09:12:12.523 INFO:teuthology.orchestra.run.smithi135.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.35880.log 2024-09-14T09:12:12.523 INFO:teuthology.orchestra.run.smithi135.stderr:/var/log/ceph/cluster1-client.admin.37505.log: gzip -5 0.0% --verbose -- /var/log/ceph/cluster1-client.admin.32340.log 2024-09-14T09:12:12.523 INFO:teuthology.orchestra.run.smithi135.stderr: -- replaced with /var/log/ceph/cluster1-client.admin.37505.log.gz 2024-09-14T09:12:12.523 INFO:teuthology.orchestra.run.smithi135.stderr:/var/log/ceph/cluster1-client.admin.35880.log: gzip 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.35880.log.gz 2024-09-14T09:12:12.523 INFO:teuthology.orchestra.run.smithi135.stderr: -5 --verbose -- /var/log/ceph/cluster1-client.admin.35751.log 2024-09-14T09:12:12.523 INFO:teuthology.orchestra.run.smithi135.stderr:/var/log/ceph/cluster1-client.admin.32340.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.32340.log.gz 2024-09-14T09:12:12.523 INFO:teuthology.orchestra.run.smithi135.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.35333.log 2024-09-14T09:12:12.523 INFO:teuthology.orchestra.run.smithi135.stderr:/var/log/ceph/cluster1-client.admin.35751.log: gzip 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.35751.log.gz 2024-09-14T09:12:12.523 INFO:teuthology.orchestra.run.smithi135.stderr: -5 --verbose -- /var/log/ceph/cluster1.audit.log 2024-09-14T09:12:12.523 INFO:teuthology.orchestra.run.smithi135.stderr:/var/log/ceph/cluster1-client.admin.35333.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.35333.log.gz 2024-09-14T09:12:12.523 INFO:teuthology.orchestra.run.smithi135.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.37545.log 2024-09-14T09:12:12.523 INFO:teuthology.orchestra.run.smithi135.stderr:/var/log/ceph/cluster1.audit.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.35291.log 2024-09-14T09:12:12.524 INFO:teuthology.orchestra.run.smithi135.stderr:/var/log/ceph/cluster1-client.admin.37545.log: 0.0%gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.35116.log 2024-09-14T09:12:12.524 INFO:teuthology.orchestra.run.smithi135.stderr: -- replaced with /var/log/ceph/cluster1-client.admin.37545.log.gz 2024-09-14T09:12:12.524 INFO:teuthology.orchestra.run.smithi135.stderr:/var/log/ceph/cluster1-client.admin.35291.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.35291.log.gz 2024-09-14T09:12:12.524 INFO:teuthology.orchestra.run.smithi135.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-osd.0.log 2024-09-14T09:12:12.524 INFO:teuthology.orchestra.run.smithi135.stderr:/var/log/ceph/cluster1-client.admin.35116.log: 0.0%gzip -- replaced with /var/log/ceph/cluster1-client.admin.35116.log.gz -5 --verbose -- /var/log/ceph/cluster1-client.admin.32161.log 2024-09-14T09:12:12.524 INFO:teuthology.orchestra.run.smithi135.stderr: 2024-09-14T09:12:12.524 INFO:teuthology.orchestra.run.smithi135.stderr:/var/log/ceph/cluster1-osd.0.log: 93.0% -- replaced with /var/log/ceph/cluster1.audit.log.gz 2024-09-14T09:12:12.524 INFO:teuthology.orchestra.run.smithi135.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.32201.log 2024-09-14T09:12:12.524 INFO:teuthology.orchestra.run.smithi135.stderr:/var/log/ceph/cluster1-client.admin.32161.log: 0.0%gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.35625.log 2024-09-14T09:12:12.524 INFO:teuthology.orchestra.run.smithi135.stderr: -- replaced with /var/log/ceph/cluster1-client.admin.32161.log.gz 2024-09-14T09:12:12.524 INFO:teuthology.orchestra.run.smithi135.stderr:/var/log/ceph/cluster1-client.admin.32201.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.32201.log.gz 2024-09-14T09:12:12.524 INFO:teuthology.orchestra.run.smithi135.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.37294.log 2024-09-14T09:12:12.524 INFO:teuthology.orchestra.run.smithi135.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1.log 2024-09-14T09:12:12.525 INFO:teuthology.orchestra.run.smithi135.stderr:/var/log/ceph/cluster1-client.admin.37294.log: /var/log/ceph/cluster1-client.admin.35625.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.35625.log.gz 2024-09-14T09:12:12.525 INFO:teuthology.orchestra.run.smithi135.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.37294.log.gz 2024-09-14T09:12:12.525 INFO:teuthology.orchestra.run.smithi135.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.35240.log 2024-09-14T09:12:12.525 INFO:teuthology.orchestra.run.smithi135.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.35771.log 2024-09-14T09:12:12.525 INFO:teuthology.orchestra.run.smithi135.stderr:/var/log/ceph/cluster1-client.admin.35240.log: /var/log/ceph/cluster1.log: 80.3% -- replaced with /var/log/ceph/cluster1-client.admin.35240.log.gz 2024-09-14T09:12:12.526 INFO:teuthology.orchestra.run.smithi135.stderr: 87.8% -- replaced with /var/log/ceph/cluster1.log.gz 2024-09-14T09:12:12.526 INFO:teuthology.orchestra.run.smithi135.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.35417.log 2024-09-14T09:12:12.526 INFO:teuthology.orchestra.run.smithi135.stderr:/var/log/ceph/cluster1-client.admin.35771.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.35771.log.gz 2024-09-14T09:12:12.526 INFO:teuthology.orchestra.run.smithi135.stderr:/var/log/ceph/cluster1-client.admin.35417.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.35417.log.gz 2024-09-14T09:12:12.538 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63037.log 2024-09-14T09:12:12.539 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73304.log 2024-09-14T09:12:12.540 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62006.log 2024-09-14T09:12:12.540 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.63037.log: /var/log/ceph/cluster2-client.mirror.73304.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45643.log 2024-09-14T09:12:12.540 INFO:teuthology.orchestra.run.smithi164.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.63037.log.gz 2024-09-14T09:12:12.540 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.62006.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45833.log 2024-09-14T09:12:12.540 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.45643.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50057.log 2024-09-14T09:12:12.541 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.45833.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.74010.log 2024-09-14T09:12:12.541 INFO:teuthology.orchestra.run.smithi164.stderr: 79.8% -- replaced with /var/log/ceph/cluster2-client.mirror.45833.log.gz 2024-09-14T09:12:12.542 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.50057.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46326.log 1.3% 2024-09-14T09:12:12.542 INFO:teuthology.orchestra.run.smithi164.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.50057.log.gz 2024-09-14T09:12:12.542 INFO:teuthology.orchestra.run.smithi164.stderr: 92.4% -- replaced with /var/log/ceph/cluster2-client.mirror.62006.log.gz 2024-09-14T09:12:12.542 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49193.log 2024-09-14T09:12:12.543 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.admin.74010.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.74010.log.gz 2024-09-14T09:12:12.543 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.46326.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50014.log 2024-09-14T09:12:12.543 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.49193.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68932.log 2024-09-14T09:12:12.543 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.50014.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47288.log 2024-09-14T09:12:12.562 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.68932.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.36548.log 2024-09-14T09:12:12.562 INFO:teuthology.orchestra.run.smithi164.stderr: 87.3%/var/log/ceph/cluster2-client.mirror.47288.log: -- replaced with /var/log/ceph/cluster1-client.mirror.68932.log.gz 2024-09-14T09:12:12.562 INFO:teuthology.orchestra.run.smithi164.stderr: 79.5% -- replaced with /var/log/ceph/cluster2-client.mirror.47288.log.gz 2024-09-14T09:12:12.562 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70417.log 2024-09-14T09:12:12.562 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose/var/log/ceph/cluster1-client.admin.36548.log: -- /var/log/ceph/cluster1-client.mirror.56859.log 2024-09-14T09:12:12.562 INFO:teuthology.orchestra.run.smithi164.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.36548.log.gz 2024-09-14T09:12:12.562 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.70417.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60691.log 2024-09-14T09:12:12.563 INFO:teuthology.orchestra.run.smithi164.stderr: 87.0% -- replaced with /var/log/ceph/cluster2-client.mirror.70417.log.gz/var/log/ceph/cluster1-client.mirror.56859.log: 2024-09-14T09:12:12.563 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60595.log 2024-09-14T09:12:12.563 INFO:teuthology.orchestra.run.smithi164.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.56859.log.gz 2024-09-14T09:12:12.563 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5/var/log/ceph/cluster1-client.mirror.60691.log: --verbose -- /var/log/ceph/cluster1-client.mirror.62162.log 2024-09-14T09:12:12.563 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.60595.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.60691.log.gz 2024-09-14T09:12:12.563 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51113.log 2024-09-14T09:12:12.563 INFO:teuthology.orchestra.run.smithi164.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.60595.log.gz 2024-09-14T09:12:12.563 INFO:teuthology.orchestra.run.smithi164.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.46326.log.gz 2024-09-14T09:12:12.563 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.62162.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59820.log 2024-09-14T09:12:12.563 INFO:teuthology.orchestra.run.smithi164.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.62162.log.gz 2024-09-14T09:12:12.563 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.51113.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68304.log 2024-09-14T09:12:12.563 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5/var/log/ceph/cluster2-client.mirror.59820.log: --verbose -- /var/log/ceph/cluster2-client.mirror.62717.log 2024-09-14T09:12:12.563 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.68304.log: gzip -5 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.68304.log.gz 2024-09-14T09:12:12.564 INFO:teuthology.orchestra.run.smithi164.stderr: 89.0% --verbose -- /var/log/ceph/cluster1-client.mirror.68394.log 2024-09-14T09:12:12.564 INFO:teuthology.orchestra.run.smithi164.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.59820.log.gz 2024-09-14T09:12:12.564 INFO:teuthology.orchestra.run.smithi164.stderr:gzip/var/log/ceph/cluster2-client.mirror.62717.log: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49390.log 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.51113.log.gz 2024-09-14T09:12:12.564 INFO:teuthology.orchestra.run.smithi164.stderr: 2024-09-14T09:12:12.564 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.68394.log: 86.9%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70873.log 2024-09-14T09:12:12.564 INFO:teuthology.orchestra.run.smithi164.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.68394.log.gz 2024-09-14T09:12:12.564 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.49390.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59888.log 2024-09-14T09:12:12.564 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.71849.log 2024-09-14T09:12:12.564 INFO:teuthology.orchestra.run.smithi164.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.49390.log.gz 2024-09-14T09:12:12.564 INFO:teuthology.orchestra.run.smithi164.stderr: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.62717.log.gz 2024-09-14T09:12:12.564 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.59888.log: /var/log/ceph/cluster2-client.mirror.70873.log: gzip -5 87.4% --verbose -- /var/log/ceph/cluster1-client.mirror.55863.log 2024-09-14T09:12:12.564 INFO:teuthology.orchestra.run.smithi164.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.59888.log.gz 2024-09-14T09:12:12.564 INFO:teuthology.orchestra.run.smithi164.stderr: 86.9% -- replaced with /var/log/ceph/cluster2-client.mirror.70873.log.gz 2024-09-14T09:12:12.565 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.71849.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64009.log 2024-09-14T09:12:12.565 INFO:teuthology.orchestra.run.smithi164.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.71849.log.gz 2024-09-14T09:12:12.565 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.55863.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69583.log 2024-09-14T09:12:12.565 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.64009.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.74446.log 2024-09-14T09:12:12.565 INFO:teuthology.orchestra.run.smithi164.stderr: 87.7%/var/log/ceph/cluster2-client.mirror.69583.log: -- replaced with /var/log/ceph/cluster2-client.mirror.64009.log.gz 2024-09-14T09:12:12.565 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48503.log 2024-09-14T09:12:12.565 INFO:teuthology.orchestra.run.smithi164.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.69583.log.gz 2024-09-14T09:12:12.565 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66046.log 2024-09-14T09:12:12.565 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.admin.74446.log: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.55863.log.gz 2024-09-14T09:12:12.565 INFO:teuthology.orchestra.run.smithi164.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.74446.log.gz 2024-09-14T09:12:12.565 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65488.log 2024-09-14T09:12:12.565 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.48503.log: /var/log/ceph/cluster2-client.mirror.66046.log: gzip 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.48503.log.gz 2024-09-14T09:12:12.565 INFO:teuthology.orchestra.run.smithi164.stderr: -5 --verbose -- /var/log/ceph/cluster1-client.admin.37365.log 2024-09-14T09:12:12.565 INFO:teuthology.orchestra.run.smithi164.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.66046.log.gz 2024-09-14T09:12:12.566 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.65488.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65839.log 2024-09-14T09:12:12.566 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.admin.37365.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.65488.log.gz 2024-09-14T09:12:12.566 INFO:teuthology.orchestra.run.smithi164.stderr: 80.3% -- replaced with /var/log/ceph/cluster1-client.admin.37365.log.gz 2024-09-14T09:12:12.566 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.47066.log 2024-09-14T09:12:12.566 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66279.log 2024-09-14T09:12:12.566 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.65839.log: /var/log/ceph/cluster1-client.mirror.1.47066.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50498.log 2024-09-14T09:12:12.566 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.66279.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.40687.log 2024-09-14T09:12:12.566 INFO:teuthology.orchestra.run.smithi164.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.66279.log.gz 2024-09-14T09:12:12.566 INFO:teuthology.orchestra.run.smithi164.stderr: 90.0%/var/log/ceph/cluster1-client.mirror.50498.log: -- replaced with /var/log/ceph/cluster2-client.mirror.65839.log.gz 2024-09-14T09:12:12.566 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41696.log 2024-09-14T09:12:12.566 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.40687.log: 87.7%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43835.log 2024-09-14T09:12:12.566 INFO:teuthology.orchestra.run.smithi164.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.50498.log.gz 2024-09-14T09:12:12.566 INFO:teuthology.orchestra.run.smithi164.stderr: 88.0% -- replaced with /var/log/ceph/cluster1-client.mirror.40687.log.gz 2024-09-14T09:12:12.567 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73788.log 2024-09-14T09:12:12.567 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.41696.log: /var/log/ceph/cluster2-client.mirror.43835.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42167.log 2024-09-14T09:12:12.567 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.73788.log: 90.3% -- replaced with /var/log/ceph/cluster2-client.mirror.73788.log.gz 2024-09-14T09:12:12.577 INFO:teuthology.orchestra.run.smithi164.stderr: 93.3% -- replaced with /var/log/ceph/cluster2-client.mirror.43835.log.gz 2024-09-14T09:12:12.577 INFO:teuthology.orchestra.run.smithi164.stderr: 93.4% -- replaced with /var/log/ceph/cluster1-client.mirror.41696.log.gz 2024-09-14T09:12:12.577 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70853.log 2024-09-14T09:12:12.577 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.42167.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.74404.log 2024-09-14T09:12:12.577 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.70853.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72290.log 2024-09-14T09:12:12.577 INFO:teuthology.orchestra.run.smithi164.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.70853.log.gz 2024-09-14T09:12:12.577 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.admin.74404.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.74404.log.gz 2024-09-14T09:12:12.577 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47029.log 2024-09-14T09:12:12.578 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56932.log 2024-09-14T09:12:12.578 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.72290.log: /var/log/ceph/cluster2-client.mirror.47029.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.72290.log.gz 2024-09-14T09:12:12.578 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46415.log 2024-09-14T09:12:12.578 INFO:teuthology.orchestra.run.smithi164.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.47029.log.gz 2024-09-14T09:12:12.578 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.56932.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.56932.log.gz 2024-09-14T09:12:12.578 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48210.log 2024-09-14T09:12:12.578 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55505.log 2024-09-14T09:12:12.578 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.46415.log: /var/log/ceph/cluster2-client.mirror.48210.log: 79.6% -- replaced with /var/log/ceph/cluster2-client.mirror.46415.log.gz 2024-09-14T09:12:12.578 INFO:teuthology.orchestra.run.smithi164.stderr: 93.4% -- replaced with /var/log/ceph/cluster1-client.mirror.42167.log.gzgzip 2024-09-14T09:12:12.579 INFO:teuthology.orchestra.run.smithi164.stderr: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59466.log 2024-09-14T09:12:12.579 INFO:teuthology.orchestra.run.smithi164.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.48210.log.gz 2024-09-14T09:12:12.579 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53980.log 2024-09-14T09:12:12.579 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.55505.log: /var/log/ceph/cluster2-client.mirror.59466.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70060.log 2024-09-14T09:12:12.579 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.53980.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.55505.log.gz 2024-09-14T09:12:12.579 INFO:teuthology.orchestra.run.smithi164.stderr: 87.3% -- replaced with /var/log/ceph/cluster2-client.mirror.59466.log.gz 2024-09-14T09:12:12.579 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62273.log 2024-09-14T09:12:12.579 INFO:teuthology.orchestra.run.smithi164.stderr: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.53980.log.gz 2024-09-14T09:12:12.579 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65695.log 2024-09-14T09:12:12.579 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.70060.log: /var/log/ceph/cluster2-client.mirror.62273.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69999.log 2024-09-14T09:12:12.579 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.65695.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.62273.log.gz 2024-09-14T09:12:12.580 INFO:teuthology.orchestra.run.smithi164.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.65695.log.gz 2024-09-14T09:12:12.580 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39010.log 2024-09-14T09:12:12.580 INFO:teuthology.orchestra.run.smithi164.stderr: 90.4%/var/log/ceph/cluster2-client.mirror.69999.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70019.log 2024-09-14T09:12:12.580 INFO:teuthology.orchestra.run.smithi164.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.70060.log.gz 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.69999.log.gz 2024-09-14T09:12:12.580 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.39010.log: 2024-09-14T09:12:12.580 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60264.log 2024-09-14T09:12:12.580 INFO:teuthology.orchestra.run.smithi164.stderr: 89.0%/var/log/ceph/cluster2-client.mirror.70019.log: -- replaced with /var/log/ceph/cluster2-client.mirror.39010.log.gz 2024-09-14T09:12:12.580 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72541.log 2024-09-14T09:12:12.580 INFO:teuthology.orchestra.run.smithi164.stderr: 87.0% -- replaced with /var/log/ceph/cluster2-client.mirror.70019.log.gz 2024-09-14T09:12:12.580 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70499.log 2024-09-14T09:12:12.580 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.60264.log: /var/log/ceph/cluster2-client.mirror.72541.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63120.log 2024-09-14T09:12:12.580 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.70499.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.72541.log.gz 2024-09-14T09:12:12.580 INFO:teuthology.orchestra.run.smithi164.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.60264.log.gz 2024-09-14T09:12:12.581 INFO:teuthology.orchestra.run.smithi164.stderr: 87.0% -- replaced with /var/log/ceph/cluster2-client.mirror.70499.log.gz 2024-09-14T09:12:12.581 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50864.log 2024-09-14T09:12:12.581 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47586.log 2024-09-14T09:12:12.581 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.63120.log: /var/log/ceph/cluster1-client.mirror.50864.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73163.log 2024-09-14T09:12:12.584 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.47586.log: 90.1% -- replaced with /var/log/ceph/cluster2-client.mirror.63120.log.gz 2024-09-14T09:12:12.584 INFO:teuthology.orchestra.run.smithi164.stderr: 92.2%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52505.log 2024-09-14T09:12:12.584 INFO:teuthology.orchestra.run.smithi164.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.50864.log.gz 2024-09-14T09:12:12.584 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.73163.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.73163.log.gz 2024-09-14T09:12:12.584 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40817.log 2024-09-14T09:12:12.584 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70242.log 2024-09-14T09:12:12.584 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.52505.log: /var/log/ceph/cluster2-client.mirror.40817.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.52505.log.gz 2024-09-14T09:12:12.584 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69770.log 2024-09-14T09:12:12.584 INFO:teuthology.orchestra.run.smithi164.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.40817.log.gz 2024-09-14T09:12:12.585 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.70242.log: 81.4% -- replaced with /var/log/ceph/cluster1-client.mirror.70242.log.gz 2024-09-14T09:12:12.585 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47325.log 2024-09-14T09:12:12.585 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48445.log 2024-09-14T09:12:12.585 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.69770.log: /var/log/ceph/cluster2-client.mirror.47325.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.69770.log.gz 2024-09-14T09:12:12.585 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43770.log 2024-09-14T09:12:12.585 INFO:teuthology.orchestra.run.smithi164.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.47325.log.gz 2024-09-14T09:12:12.594 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.48445.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60817.log 2024-09-14T09:12:12.595 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.43770.log: 88.5% -- replaced with /var/log/ceph/cluster1-client.mirror.43770.log.gz 2024-09-14T09:12:12.597 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63078.log 2024-09-14T09:12:12.598 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.60817.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.60817.log.gz 2024-09-14T09:12:12.599 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43984.log 2024-09-14T09:12:12.599 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.63078.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.63078.log.gz 2024-09-14T09:12:12.600 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70622.log 2024-09-14T09:12:12.606 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.43984.log: 93.4% -- replaced with /var/log/ceph/cluster1-client.mirror.43984.log.gz 2024-09-14T09:12:12.606 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47367.log 2024-09-14T09:12:12.608 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.70622.log: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.70622.log.gz 2024-09-14T09:12:12.608 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43855.log 2024-09-14T09:12:12.608 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.47367.log: 79.6% -- replaced with /var/log/ceph/cluster2-client.mirror.47367.log.gz 2024-09-14T09:12:12.609 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57700.log 2024-09-14T09:12:12.615 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.43855.log: 93.3% -- replaced with /var/log/ceph/cluster1-client.mirror.43855.log.gz 2024-09-14T09:12:12.615 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72061.log 2024-09-14T09:12:12.615 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.57700.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.57700.log.gz 2024-09-14T09:12:12.616 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45163.log 2024-09-14T09:12:12.617 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.72061.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.72061.log.gz 2024-09-14T09:12:12.617 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49568.log 2024-09-14T09:12:12.618 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.45163.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.45163.log.gz 2024-09-14T09:12:12.620 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58467.log 2024-09-14T09:12:12.620 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.49568.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.49568.log.gz 2024-09-14T09:12:12.620 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54993.log 2024-09-14T09:12:12.621 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.58467.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.58467.log.gz 2024-09-14T09:12:12.621 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44652.log 2024-09-14T09:12:12.622 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.54993.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.54993.log.gz 2024-09-14T09:12:12.622 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56323.log 2024-09-14T09:12:12.623 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.44652.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.44652.log.gz 2024-09-14T09:12:12.624 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58238.log 2024-09-14T09:12:12.624 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.56323.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.56323.log.gz 2024-09-14T09:12:12.625 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54021.log 2024-09-14T09:12:12.625 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.58238.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.58238.log.gz 2024-09-14T09:12:12.626 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41445.log 2024-09-14T09:12:12.628 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.54021.log: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.54021.log.gz 2024-09-14T09:12:12.628 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69917.log 2024-09-14T09:12:12.630 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.41445.log: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.41445.log.gz 2024-09-14T09:12:12.630 INFO:teuthology.orchestra.run.smithi164.stderr: 93.3% -- replaced with /var/log/ceph/cluster1-client.mirror.1.47066.log.gz 2024-09-14T09:12:12.630 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43216.log 2024-09-14T09:12:12.630 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.69917.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53364.log 2024-09-14T09:12:12.631 INFO:teuthology.orchestra.run.smithi164.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.69917.log.gz/var/log/ceph/cluster2-client.mirror.43216.log: 2024-09-14T09:12:12.643 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49972.log 2024-09-14T09:12:12.644 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.53364.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.53364.log.gz 2024-09-14T09:12:12.653 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67359.log 2024-09-14T09:12:12.654 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.49972.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.49972.log.gz 2024-09-14T09:12:12.658 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42520.log 2024-09-14T09:12:12.659 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.67359.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.67359.log.gz 2024-09-14T09:12:12.659 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68169.log 2024-09-14T09:12:12.661 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.42520.log: 89.1% -- replaced with /var/log/ceph/cluster1-client.mirror.42520.log.gz 2024-09-14T09:12:12.661 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61249.log 2024-09-14T09:12:12.662 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.68169.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.68169.log.gz 2024-09-14T09:12:12.662 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40425.log 2024-09-14T09:12:12.663 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.61249.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.61249.log.gz 2024-09-14T09:12:12.663 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67832.log 2024-09-14T09:12:12.663 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.40425.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.40425.log.gz 2024-09-14T09:12:12.666 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.37518.log 2024-09-14T09:12:12.666 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.67832.log: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.67832.log.gz 2024-09-14T09:12:12.666 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58102.log 2024-09-14T09:12:12.666 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.admin.37518.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.37518.log.gz 2024-09-14T09:12:12.666 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46831.log 2024-09-14T09:12:12.667 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.58102.log: 92.2% -- replaced with /var/log/ceph/cluster1-client.mirror.58102.log.gz 2024-09-14T09:12:12.668 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.40141.log 2024-09-14T09:12:12.681 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.46831.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36907.log 2024-09-14T09:12:12.682 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.40141.log: 88.0% -- replaced with /var/log/ceph/cluster1-client.mirror.40141.log.gz 2024-09-14T09:12:12.694 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66909.log 2024-09-14T09:12:12.704 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.36907.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70702.log 2024-09-14T09:12:12.713 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.66909.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68057.log 2024-09-14T09:12:12.716 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.70702.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.66909.log.gz 2024-09-14T09:12:12.725 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.62925.log 2024-09-14T09:12:12.726 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.68057.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.68057.log.gz 2024-09-14T09:12:12.737 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69876.log 2024-09-14T09:12:12.737 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.62925.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.62925.log.gz 2024-09-14T09:12:12.747 INFO:teuthology.orchestra.run.smithi164.stderr: 93.7%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72122.log 2024-09-14T09:12:12.747 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.69876.log: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.48445.log.gz 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.69876.log.gz 2024-09-14T09:12:12.747 INFO:teuthology.orchestra.run.smithi164.stderr: 2024-09-14T09:12:12.748 INFO:teuthology.orchestra.run.smithi164.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.47586.log.gz 2024-09-14T09:12:12.760 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61668.log 2024-09-14T09:12:12.761 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.72122.log: 87.0% -- replaced with /var/log/ceph/cluster2-client.mirror.72122.log.gz 2024-09-14T09:12:12.777 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51352.log 2024-09-14T09:12:12.778 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.61668.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.61668.log.gz 2024-09-14T09:12:12.790 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55135.log 2024-09-14T09:12:12.791 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.51352.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.51352.log.gz 2024-09-14T09:12:12.804 INFO:teuthology.orchestra.run.smithi164.stderr: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.50014.log.gz 2024-09-14T09:12:12.804 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.40254.log 2024-09-14T09:12:12.804 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.55135.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46933.log 2024-09-14T09:12:12.805 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.40254.log: 87.2% -- replaced with /var/log/ceph/cluster1-client.mirror.55135.log.gz 2024-09-14T09:12:12.810 INFO:teuthology.orchestra.run.smithi164.stderr: 93.3% -- replaced with /var/log/ceph/cluster1-client.mirror.40254.log.gz 2024-09-14T09:12:12.813 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63596.log 2024-09-14T09:12:12.814 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.46933.log: 79.6% -- replaced with /var/log/ceph/cluster2-client.mirror.46933.log.gz 2024-09-14T09:12:12.814 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43152.log 2024-09-14T09:12:12.814 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.63596.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.63596.log.gz 2024-09-14T09:12:12.814 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62829.log 2024-09-14T09:12:12.820 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.43152.log: 93.4% -- replaced with /var/log/ceph/cluster2-client.mirror.43152.log.gz 2024-09-14T09:12:12.820 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58732.log 2024-09-14T09:12:12.820 INFO:teuthology.orchestra.run.smithi135.stderr: 91.2% -- replaced with /var/log/ceph/cluster1-mgr.x.log.gz 2024-09-14T09:12:12.824 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.62829.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.62829.log.gz 2024-09-14T09:12:12.824 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72455.log 2024-09-14T09:12:12.824 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.58732.log: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.58732.log.gz 2024-09-14T09:12:12.824 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65054.log 2024-09-14T09:12:12.824 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.72455.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.72455.log.gz 2024-09-14T09:12:12.824 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65612.log 2024-09-14T09:12:12.825 INFO:teuthology.orchestra.run.smithi135.stderr: 91.2% -- replaced with /var/log/ceph/cluster2-mgr.x.log.gz 2024-09-14T09:12:12.827 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.65054.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.65054.log.gz 2024-09-14T09:12:12.827 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53908.log 2024-09-14T09:12:12.827 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.65612.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.65612.log.gz 2024-09-14T09:12:12.827 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66258.log 2024-09-14T09:12:12.827 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.53908.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.53908.log.gz 2024-09-14T09:12:12.830 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64951.log 2024-09-14T09:12:12.830 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.66258.log: 1.3% -- replaced with /var/log/ceph/cluster1-client.mirror.66258.log.gz 2024-09-14T09:12:12.830 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54882.log 2024-09-14T09:12:12.830 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.64951.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.64951.log.gz 2024-09-14T09:12:12.830 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53719.log 2024-09-14T09:12:12.832 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.54882.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.54882.log.gz 2024-09-14T09:12:12.832 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50683.log 2024-09-14T09:12:12.833 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.53719.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.53719.log.gz 2024-09-14T09:12:12.833 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46089.log 2024-09-14T09:12:12.834 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.50683.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.50683.log.gz 2024-09-14T09:12:12.834 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48800.log 2024-09-14T09:12:12.835 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.46089.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.46089.log.gz 2024-09-14T09:12:12.836 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57194.log 2024-09-14T09:12:12.837 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.48800.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.48800.log.gz 2024-09-14T09:12:12.838 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.32918.log 2024-09-14T09:12:12.841 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.57194.log: 92.2% -- replaced with /var/log/ceph/cluster1-client.mirror.57194.log.gz 2024-09-14T09:12:12.841 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61889.log 2024-09-14T09:12:12.841 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.admin.32918.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.32918.log.gz 2024-09-14T09:12:12.842 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46541.log 2024-09-14T09:12:12.844 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.61889.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.61889.log.gz 2024-09-14T09:12:12.844 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59684.log 2024-09-14T09:12:12.844 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.46541.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.46541.log.gz 2024-09-14T09:12:12.844 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49893.log 2024-09-14T09:12:12.846 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.59684.log: 92.0% -- replaced with /var/log/ceph/cluster1-client.mirror.59684.log.gz 2024-09-14T09:12:12.846 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61458.log 2024-09-14T09:12:12.848 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.49893.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.49893.log.gz 2024-09-14T09:12:12.848 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45892.log 2024-09-14T09:12:12.851 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.61458.log: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.61458.log.gz 2024-09-14T09:12:12.851 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54814.log 2024-09-14T09:12:12.859 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.45892.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64174.log 2024-09-14T09:12:12.873 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.54814.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.54814.log.gz 2024-09-14T09:12:12.886 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41783.log 2024-09-14T09:12:12.887 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.64174.log: 90.1% -- replaced with /var/log/ceph/cluster2-client.mirror.64174.log.gz 2024-09-14T09:12:12.888 INFO:teuthology.orchestra.run.smithi164.stderr: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.43216.log.gz 2024-09-14T09:12:12.889 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63988.log 2024-09-14T09:12:12.891 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.41783.log: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.49193.log.gz 2024-09-14T09:12:12.892 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49250.log 2024-09-14T09:12:12.893 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.63988.log: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.63988.log.gz 2024-09-14T09:12:12.899 INFO:teuthology.orchestra.run.smithi164.stderr: 93.6% -- replaced with /var/log/ceph/cluster2-client.mirror.45643.log.gz 2024-09-14T09:12:12.901 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69728.log 2024-09-14T09:12:12.901 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.49250.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.72868.log 2024-09-14T09:12:12.901 INFO:teuthology.orchestra.run.smithi164.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.49250.log.gz 2024-09-14T09:12:12.902 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.69728.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.69728.log.gz 2024-09-14T09:12:12.902 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45771.log 2024-09-14T09:12:12.907 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose --/var/log/ceph/cluster1-client.mirror.72868.log: /var/log/ceph/cluster2-client.mirror.63658.log 2024-09-14T09:12:12.907 INFO:teuthology.orchestra.run.smithi164.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.72868.log.gz 2024-09-14T09:12:12.907 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.45771.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46230.log 2024-09-14T09:12:12.907 INFO:teuthology.orchestra.run.smithi164.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.45771.log.gz 2024-09-14T09:12:12.907 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.63658.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.63658.log.gz 2024-09-14T09:12:12.907 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50078.log 2024-09-14T09:12:12.907 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.46230.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66684.log 2024-09-14T09:12:12.907 INFO:teuthology.orchestra.run.smithi164.stderr: 79.8% -- replaced with /var/log/ceph/cluster2-client.mirror.46230.log.gz 2024-09-14T09:12:12.907 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.50078.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43898.log 2024-09-14T09:12:12.907 INFO:teuthology.orchestra.run.smithi164.stderr: 1.3% -- replaced with /var/log/ceph/cluster1-client.mirror.50078.log.gz 2024-09-14T09:12:12.907 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.66684.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.66684.log.gz 2024-09-14T09:12:12.908 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57057.log 2024-09-14T09:12:12.908 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.43898.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.62210.log 2024-09-14T09:12:12.916 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.57057.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.57057.log.gz 2024-09-14T09:12:12.916 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58286.log 2024-09-14T09:12:12.917 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.62210.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.62210.log.gz 2024-09-14T09:12:12.917 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52999.log 2024-09-14T09:12:12.917 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.58286.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.58286.log.gz 2024-09-14T09:12:12.917 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39434.log 2024-09-14T09:12:12.917 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.52999.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.52999.log.gz 2024-09-14T09:12:12.917 INFO:teuthology.orchestra.run.smithi164.stderr: 93.2% -- replaced with /var/log/ceph/cluster1-client.mirror.43898.log.gz 2024-09-14T09:12:12.917 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58082.log 2024-09-14T09:12:12.917 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58013.log 2024-09-14T09:12:12.917 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.58082.log: /var/log/ceph/cluster2-client.mirror.39434.log: 74.1% -- replaced with /var/log/ceph/cluster2-client.mirror.39434.log.gz 2024-09-14T09:12:12.917 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54181.log 2024-09-14T09:12:12.917 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.58013.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.58013.log.gz 2024-09-14T09:12:12.918 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36928.log 2024-09-14T09:12:12.918 INFO:teuthology.orchestra.run.smithi164.stderr: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.58082.log.gz 2024-09-14T09:12:12.918 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.54181.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46168.log 2024-09-14T09:12:12.918 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.36928.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.54181.log.gz 2024-09-14T09:12:12.918 INFO:teuthology.orchestra.run.smithi164.stderr: 82.2% -- replaced with /var/log/ceph/cluster2-client.mirror.36928.log.gz 2024-09-14T09:12:12.918 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53047.log 2024-09-14T09:12:12.918 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41631.log 2024-09-14T09:12:12.918 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.46168.log: /var/log/ceph/cluster1-client.mirror.53047.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.53047.log.gz 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.46168.log.gz 2024-09-14T09:12:12.918 INFO:teuthology.orchestra.run.smithi164.stderr: 2024-09-14T09:12:12.928 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49469.log 2024-09-14T09:12:12.929 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose/var/log/ceph/cluster1-client.mirror.41631.log: -- /var/log/ceph/cluster2-client.admin.32675.log 2024-09-14T09:12:12.929 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.49469.log: 88.5% -- replaced with /var/log/ceph/cluster1-client.mirror.41631.log.gz 2024-09-14T09:12:12.929 INFO:teuthology.orchestra.run.smithi164.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.49469.log.gz 2024-09-14T09:12:12.929 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45032.log 2024-09-14T09:12:12.929 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.admin.32675.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.32675.log.gzgzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63947.log 2024-09-14T09:12:12.929 INFO:teuthology.orchestra.run.smithi164.stderr: 2024-09-14T09:12:12.929 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47846.log 2024-09-14T09:12:12.929 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.45032.log: /var/log/ceph/cluster2-client.mirror.63947.log: 87.7% 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.45032.log.gz 2024-09-14T09:12:12.929 INFO:teuthology.orchestra.run.smithi164.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.63947.log.gz 2024-09-14T09:12:12.929 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48006.log 2024-09-14T09:12:12.929 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose --/var/log/ceph/cluster2-client.mirror.47846.log: /var/log/ceph/cluster1-client.admin.38716.log 2024-09-14T09:12:12.929 INFO:teuthology.orchestra.run.smithi164.stderr: 79.9% -- replaced with /var/log/ceph/cluster2-client.mirror.47846.log.gz 2024-09-14T09:12:12.930 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.48006.log: 79.6% -- replaced with /var/log/ceph/cluster2-client.mirror.48006.log.gz 2024-09-14T09:12:12.930 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46009.log 2024-09-14T09:12:12.930 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.admin.38716.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70376.log 2024-09-14T09:12:12.930 INFO:teuthology.orchestra.run.smithi164.stderr: 87.1% -- replaced with /var/log/ceph/cluster1-client.admin.38716.log.gz 2024-09-14T09:12:12.930 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.46009.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45205.log 2024-09-14T09:12:12.930 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.70376.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.46009.log.gz 2024-09-14T09:12:12.930 INFO:teuthology.orchestra.run.smithi164.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.70376.log.gz 2024-09-14T09:12:12.930 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68034.log 2024-09-14T09:12:12.930 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58349.log 2024-09-14T09:12:12.930 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.45205.log: /var/log/ceph/cluster1-client.mirror.68034.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.68034.log.gz 2024-09-14T09:12:12.930 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69061.log 2024-09-14T09:12:12.930 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.58349.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.58349.log.gz 2024-09-14T09:12:12.931 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55200.log 2024-09-14T09:12:12.931 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.69061.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.69061.log.gz 2024-09-14T09:12:12.931 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46912.log 2024-09-14T09:12:12.932 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.55200.log: 92.2% -- replaced with /var/log/ceph/cluster1-client.mirror.55200.log.gz 2024-09-14T09:12:12.932 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71893.log 2024-09-14T09:12:12.940 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.46912.log: 94.0% -- replaced with /var/log/ceph/cluster2-client.mirror.36907.log.gz 2024-09-14T09:12:12.940 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48583.log 2024-09-14T09:12:12.940 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59086.log 2024-09-14T09:12:12.940 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.48583.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.48583.log.gz 2024-09-14T09:12:12.942 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.71893.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.71893.log.gz 2024-09-14T09:12:12.942 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66954.log 2024-09-14T09:12:12.943 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.59086.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.59086.log.gz 2024-09-14T09:12:12.943 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57792.log 2024-09-14T09:12:12.944 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.66954.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.66954.log.gz 2024-09-14T09:12:12.945 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65178.log 2024-09-14T09:12:12.946 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.57792.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.57792.log.gz 2024-09-14T09:12:12.946 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68237.log 2024-09-14T09:12:12.948 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.65178.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.65178.log.gz 2024-09-14T09:12:12.948 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45053.log 2024-09-14T09:12:12.948 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.68237.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.68237.log.gz 2024-09-14T09:12:12.948 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39213.log 2024-09-14T09:12:12.949 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.45053.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.45053.log.gz 2024-09-14T09:12:12.950 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60841.log 2024-09-14T09:12:12.951 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.39213.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.39213.log.gz 2024-09-14T09:12:12.951 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52979.log 2024-09-14T09:12:12.952 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.60841.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.60841.log.gz 2024-09-14T09:12:12.953 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60467.log 2024-09-14T09:12:12.953 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.52979.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.52979.log.gz 2024-09-14T09:12:12.954 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63368.log 2024-09-14T09:12:12.956 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.60467.log: 92.3% -- replaced with /var/log/ceph/cluster2-client.mirror.60467.log.gz 2024-09-14T09:12:12.957 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58214.log 2024-09-14T09:12:12.959 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.63368.log: 90.1% -- replaced with /var/log/ceph/cluster2-client.mirror.63368.log.gz 2024-09-14T09:12:12.959 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73538.log 2024-09-14T09:12:12.960 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.58214.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.58214.log.gz 2024-09-14T09:12:12.960 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48605.log 2024-09-14T09:12:12.961 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.73538.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.73538.log.gz 2024-09-14T09:12:12.962 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51217.log 2024-09-14T09:12:12.962 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.48605.log: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.73304.log.gz 2024-09-14T09:12:12.963 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.39078.log 2024-09-14T09:12:12.964 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.51217.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.51217.log.gz 2024-09-14T09:12:12.964 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60958.log 2024-09-14T09:12:12.965 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.39078.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.39078.log.gz 2024-09-14T09:12:12.965 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63864.log 2024-09-14T09:12:12.967 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.60958.log: 92.2% -- replaced with /var/log/ceph/cluster2-client.mirror.60958.log.gz 2024-09-14T09:12:12.967 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56188.log 2024-09-14T09:12:12.969 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.63864.log: 90.2% -- replaced with /var/log/ceph/cluster2-client.mirror.63864.log.gz 2024-09-14T09:12:12.969 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59396.log 2024-09-14T09:12:12.970 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.56188.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.56188.log.gz 2024-09-14T09:12:12.970 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67989.log 2024-09-14T09:12:12.973 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.59396.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.59396.log.gz 2024-09-14T09:12:12.973 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73596.log 2024-09-14T09:12:12.973 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.67989.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.67989.log.gz 2024-09-14T09:12:12.974 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.39895.log 2024-09-14T09:12:12.974 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.73596.log: 73.7% -- replaced with /var/log/ceph/cluster2-client.mirror.73596.log.gz 2024-09-14T09:12:12.974 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56144.log 2024-09-14T09:12:12.975 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.admin.39895.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.39895.log.gz 2024-09-14T09:12:12.975 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48642.log 2024-09-14T09:12:12.976 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.56144.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.56144.log.gz 2024-09-14T09:12:12.976 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64661.log 2024-09-14T09:12:12.978 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.48642.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.48642.log.gz 2024-09-14T09:12:12.978 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46474.log 2024-09-14T09:12:12.980 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.64661.log: 90.1% -- replaced with /var/log/ceph/cluster2-client.mirror.64661.log.gz 2024-09-14T09:12:12.980 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65633.log 2024-09-14T09:12:13.000 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.46474.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53340.log 2024-09-14T09:12:13.001 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.65633.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.65633.log.gz 2024-09-14T09:12:13.009 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71605.log 2024-09-14T09:12:13.009 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.53340.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.53340.log.gz 2024-09-14T09:12:13.021 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51988.log 2024-09-14T09:12:13.022 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.71605.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.71605.log.gz 2024-09-14T09:12:13.028 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53160.log 2024-09-14T09:12:13.028 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.51988.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.51988.log.gz 2024-09-14T09:12:13.037 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49432.log 2024-09-14T09:12:13.041 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.53160.log: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.45892.log.gz 2024-09-14T09:12:13.044 INFO:teuthology.orchestra.run.smithi164.stderr: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.53160.log.gz 2024-09-14T09:12:13.044 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59446.log 2024-09-14T09:12:13.057 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.49432.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60242.log 2024-09-14T09:12:13.058 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.59446.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.59446.log.gz 2024-09-14T09:12:13.071 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69937.log 2024-09-14T09:12:13.072 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.60242.log: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.60242.log.gz 2024-09-14T09:12:13.082 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61207.log 2024-09-14T09:12:13.083 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.69937.log: 87.0% -- replaced with /var/log/ceph/cluster2-client.mirror.69937.log.gz 2024-09-14T09:12:13.083 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55485.log 2024-09-14T09:12:13.093 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.61207.log: 92.4%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63265.log 2024-09-14T09:12:13.095 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.55485.log: 88.9% 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.46831.log.gz 2024-09-14T09:12:13.096 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61339.log 2024-09-14T09:12:13.097 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.63265.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.63265.log.gz 2024-09-14T09:12:13.099 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58491.log 2024-09-14T09:12:13.099 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.61339.log: -- replaced with /var/log/ceph/cluster2-client.mirror.61207.log.gz 2024-09-14T09:12:13.099 INFO:teuthology.orchestra.run.smithi164.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.61339.log.gz 2024-09-14T09:12:13.099 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69248.log 2024-09-14T09:12:13.100 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.58491.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.58491.log.gz 2024-09-14T09:12:13.100 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70811.log 2024-09-14T09:12:13.104 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.69248.log: 87.3% -- replaced with /var/log/ceph/cluster2-client.mirror.55485.log.gz 2024-09-14T09:12:13.104 INFO:teuthology.orchestra.run.smithi164.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.69248.log.gz 2024-09-14T09:12:13.104 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68349.log 2024-09-14T09:12:13.104 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.70811.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.70811.log.gz 2024-09-14T09:12:13.105 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73224.log 2024-09-14T09:12:13.105 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.68349.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.68349.log.gz 2024-09-14T09:12:13.105 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.39868.log 2024-09-14T09:12:13.106 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.73224.log: 90.3% -- replaced with /var/log/ceph/cluster2-client.mirror.73224.log.gz 2024-09-14T09:12:13.106 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.45279.log 2024-09-14T09:12:13.107 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.admin.39868.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.39868.log.gz 2024-09-14T09:12:13.108 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50751.log 2024-09-14T09:12:13.127 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.0.45279.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48525.log 2024-09-14T09:12:13.127 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.50751.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.50751.log.gz 2024-09-14T09:12:13.134 INFO:teuthology.orchestra.run.smithi164.stderr: 92.8%gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61413.log 2024-09-14T09:12:13.140 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.48525.log: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.70702.log.gz 2024-09-14T09:12:13.140 INFO:teuthology.orchestra.run.smithi164.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.0.45279.log.gz 2024-09-14T09:12:13.141 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41939.log 2024-09-14T09:12:13.143 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.61413.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.61413.log.gz 2024-09-14T09:12:13.143 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54268.log 2024-09-14T09:12:13.143 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.41939.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.41939.log.gz 2024-09-14T09:12:13.143 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65984.log 2024-09-14T09:12:13.146 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.54268.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.54268.log.gz 2024-09-14T09:12:13.146 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45791.log 2024-09-14T09:12:13.146 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.65984.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.65984.log.gz 2024-09-14T09:12:13.146 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38742.log 2024-09-14T09:12:13.147 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.45791.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.45791.log.gz 2024-09-14T09:12:13.147 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.32715.log 2024-09-14T09:12:13.150 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.38742.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.38742.log.gz 2024-09-14T09:12:13.150 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.39344.log 2024-09-14T09:12:13.150 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.admin.32715.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.32715.log.gz 2024-09-14T09:12:13.150 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44480.log 2024-09-14T09:12:13.151 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.admin.39344.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.39344.log.gz 2024-09-14T09:12:13.151 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66662.log 2024-09-14T09:12:13.156 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.44480.log: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.44480.log.gz 2024-09-14T09:12:13.158 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54834.log 2024-09-14T09:12:13.158 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.66662.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.66662.log.gz 2024-09-14T09:12:13.158 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68757.log 2024-09-14T09:12:13.160 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.54834.log: 87.3% -- replaced with /var/log/ceph/cluster2-client.mirror.54834.log.gz 2024-09-14T09:12:13.160 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.41847.log 2024-09-14T09:12:13.164 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.68757.log: 87.2% -- replaced with /var/log/ceph/cluster1-client.mirror.68757.log.gz 2024-09-14T09:12:13.164 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57298.log 2024-09-14T09:12:13.164 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.admin.41847.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.41847.log.gz 2024-09-14T09:12:13.164 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71564.log 2024-09-14T09:12:13.164 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.57298.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.57298.log.gz 2024-09-14T09:12:13.164 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63678.log 2024-09-14T09:12:13.166 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.71564.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.71564.log.gz 2024-09-14T09:12:13.166 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57539.log 2024-09-14T09:12:13.166 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.63678.log: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.63678.log.gz 2024-09-14T09:12:13.167 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53427.log 2024-09-14T09:12:13.171 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.57539.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.57539.log.gz 2024-09-14T09:12:13.171 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55284.log 2024-09-14T09:12:13.171 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.53427.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.53427.log.gz 2024-09-14T09:12:13.172 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63430.log 2024-09-14T09:12:13.172 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.55284.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.55284.log.gz 2024-09-14T09:12:13.172 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59490.log 2024-09-14T09:12:13.174 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.63430.log: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.63430.log.gz 2024-09-14T09:12:13.174 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54629.log 2024-09-14T09:12:13.175 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.59490.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.59490.log.gz 2024-09-14T09:12:13.175 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42405.log 2024-09-14T09:12:13.176 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.54629.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.54629.log.gz 2024-09-14T09:12:13.177 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45121.log 2024-09-14T09:12:13.177 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.42405.log: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.42405.log.gz 2024-09-14T09:12:13.178 INFO:teuthology.orchestra.run.smithi164.stderr: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.45205.log.gz 2024-09-14T09:12:13.178 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45971.log 2024-09-14T09:12:13.183 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5/var/log/ceph/cluster1-client.mirror.45121.log: --verbose -- /var/log/ceph/cluster1-client.mirror.55380.log 2024-09-14T09:12:13.183 INFO:teuthology.orchestra.run.smithi164.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.45121.log.gz 2024-09-14T09:12:13.183 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.45971.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68439.log 2024-09-14T09:12:13.183 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.55380.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.55380.log.gz 2024-09-14T09:12:13.183 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68214.log 2024-09-14T09:12:13.183 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.68439.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.68439.log.gz 2024-09-14T09:12:13.183 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50886.log 2024-09-14T09:12:13.183 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.68214.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.68214.log.gz 2024-09-14T09:12:13.183 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45850.log 2024-09-14T09:12:13.186 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.50886.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.50886.log.gz 2024-09-14T09:12:13.186 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58307.log 2024-09-14T09:12:13.186 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.45850.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.45850.log.gz 2024-09-14T09:12:13.186 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72977.log 2024-09-14T09:12:13.187 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.58307.log: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.58307.log.gz 2024-09-14T09:12:13.188 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46109.log 2024-09-14T09:12:13.189 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.72977.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.72977.log.gz 2024-09-14T09:12:13.190 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58881.log 2024-09-14T09:12:13.191 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.46109.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.46109.log.gz 2024-09-14T09:12:13.191 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57650.log 2024-09-14T09:12:13.191 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.58881.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.58881.log.gz 2024-09-14T09:12:13.192 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67562.log 2024-09-14T09:12:13.193 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.57650.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.57650.log.gz 2024-09-14T09:12:13.194 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73282.log 2024-09-14T09:12:13.195 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.67562.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.67562.log.gz 2024-09-14T09:12:13.195 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67382.log 2024-09-14T09:12:13.195 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.73282.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.73282.log.gz 2024-09-14T09:12:13.197 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47384.log 2024-09-14T09:12:13.197 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.67382.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.67382.log.gz 2024-09-14T09:12:13.197 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44852.log 2024-09-14T09:12:13.199 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.47384.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.47384.log.gz 2024-09-14T09:12:13.199 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66752.log 2024-09-14T09:12:13.200 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.44852.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.44852.log.gz 2024-09-14T09:12:13.200 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67629.log 2024-09-14T09:12:13.201 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.66752.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.66752.log.gz 2024-09-14T09:12:13.202 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58515.log 2024-09-14T09:12:13.203 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.67629.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.67629.log.gz 2024-09-14T09:12:13.203 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72390.log 2024-09-14T09:12:13.204 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.58515.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.58515.log.gz 2024-09-14T09:12:13.204 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70540.log 2024-09-14T09:12:13.216 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.72390.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.62251.log 2024-09-14T09:12:13.217 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.70540.log: 86.9% -- replaced with /var/log/ceph/cluster2-client.mirror.70540.log.gz 2024-09-14T09:12:13.230 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.39665.log 2024-09-14T09:12:13.232 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.62251.log: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.62251.log.gz 2024-09-14T09:12:13.248 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56723.log 2024-09-14T09:12:13.248 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.39665.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.39665.log.gz 2024-09-14T09:12:13.251 INFO:teuthology.orchestra.run.smithi164.stderr: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.41783.log.gz 2024-09-14T09:12:13.252 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68507.log 2024-09-14T09:12:13.254 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.56723.log: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.56723.log.gz 2024-09-14T09:12:13.255 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65281.log 2024-09-14T09:12:13.256 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.68507.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.68507.log.gz 2024-09-14T09:12:13.256 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58061.log 2024-09-14T09:12:13.257 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.65281.log: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.65281.log.gz 2024-09-14T09:12:13.257 INFO:teuthology.orchestra.run.smithi164.stderr: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.46912.log.gzgzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66347.log 2024-09-14T09:12:13.257 INFO:teuthology.orchestra.run.smithi164.stderr: 2024-09-14T09:12:13.258 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.58061.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.58061.log.gz 2024-09-14T09:12:13.258 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67404.log 2024-09-14T09:12:13.258 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.66347.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.46326.log 2024-09-14T09:12:13.272 INFO:teuthology.orchestra.run.smithi164.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.66347.log.gz 2024-09-14T09:12:13.272 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.67404.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.67404.log.gz 2024-09-14T09:12:13.272 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56084.log 2024-09-14T09:12:13.272 INFO:teuthology.orchestra.run.smithi164.stderr: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.49432.log.gz 2024-09-14T09:12:13.272 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.1.46326.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66594.log 2024-09-14T09:12:13.272 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.56084.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56019.log 2024-09-14T09:12:13.272 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.66594.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.66594.log.gz 2024-09-14T09:12:13.272 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46151.log 2024-09-14T09:12:13.272 INFO:teuthology.orchestra.run.smithi164.stderr: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.56084.log.gz 2024-09-14T09:12:13.272 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.56019.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66999.log 2024-09-14T09:12:13.273 INFO:teuthology.orchestra.run.smithi164.stderr: 87.4%/var/log/ceph/cluster2-client.mirror.46151.log: -- replaced with /var/log/ceph/cluster1-client.mirror.56019.log.gz 2024-09-14T09:12:13.273 INFO:teuthology.orchestra.run.smithi164.stderr: 79.8% -- replaced with /var/log/ceph/cluster2-client.mirror.46151.log.gz 2024-09-14T09:12:13.273 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64909.log 2024-09-14T09:12:13.273 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.66999.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49876.log 2024-09-14T09:12:13.273 INFO:teuthology.orchestra.run.smithi164.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.66999.log.gz 2024-09-14T09:12:13.273 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.64909.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60619.log 2024-09-14T09:12:13.273 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.49876.log: 79.6% -- replaced with /var/log/ceph/cluster2-client.mirror.49876.log.gz 2024-09-14T09:12:13.273 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43478.log 2024-09-14T09:12:13.273 INFO:teuthology.orchestra.run.smithi164.stderr: 90.1%/var/log/ceph/cluster1-client.mirror.60619.log: -- replaced with /var/log/ceph/cluster2-client.mirror.64909.log.gz 2024-09-14T09:12:13.273 INFO:teuthology.orchestra.run.smithi164.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.60619.log.gzgzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41987.log 2024-09-14T09:12:13.273 INFO:teuthology.orchestra.run.smithi164.stderr: 2024-09-14T09:12:13.273 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.39617.log 2024-09-14T09:12:13.273 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.41987.log: /var/log/ceph/cluster1-client.mirror.43478.log: 88.9% 89.3% -- replaced with /var/log/ceph/cluster1-client.mirror.43478.log.gz 2024-09-14T09:12:13.274 INFO:teuthology.orchestra.run.smithi164.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.41987.log.gz 2024-09-14T09:12:13.274 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61138.log 2024-09-14T09:12:13.274 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.39617.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61315.log 2024-09-14T09:12:13.274 INFO:teuthology.orchestra.run.smithi164.stderr: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.39617.log.gz 2024-09-14T09:12:13.274 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.61138.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44027.log 2024-09-14T09:12:13.274 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.61315.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.61138.log.gz 2024-09-14T09:12:13.274 INFO:teuthology.orchestra.run.smithi164.stderr: 87.4%gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57768.log 2024-09-14T09:12:13.274 INFO:teuthology.orchestra.run.smithi164.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.61315.log.gz 2024-09-14T09:12:13.274 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.1.46326.log.gz 2024-09-14T09:12:13.274 INFO:teuthology.orchestra.run.smithi164.stderr: /var/log/ceph/cluster2-client.mirror.63554.log 2024-09-14T09:12:13.274 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.44027.log: /var/log/ceph/cluster1-client.mirror.57768.log: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.48605.log.gz 2024-09-14T09:12:13.274 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73100.log 2024-09-14T09:12:13.274 INFO:teuthology.orchestra.run.smithi164.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.57768.log.gz 2024-09-14T09:12:13.274 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.63554.log: 90.1% -- replaced with /var/log/ceph/cluster2-client.mirror.63554.log.gz 2024-09-14T09:12:13.275 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.37300.log 2024-09-14T09:12:13.291 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.73100.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44609.log 2024-09-14T09:12:13.291 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.admin.37300.log: 87.0% -- replaced with /var/log/ceph/cluster2-client.mirror.73100.log.gz 2024-09-14T09:12:13.292 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.41419.log 2024-09-14T09:12:13.292 INFO:teuthology.orchestra.run.smithi164.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.37300.log.gz 2024-09-14T09:12:13.292 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose/var/log/ceph/cluster2-client.mirror.44609.log: -- /var/log/ceph/cluster1-client.mirror.37399.log 2024-09-14T09:12:13.292 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.admin.41419.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58660.log 2024-09-14T09:12:13.292 INFO:teuthology.orchestra.run.smithi164.stderr: 87.6% 93.4% -- replaced with /var/log/ceph/cluster1-client.mirror.44027.log.gz 2024-09-14T09:12:13.292 INFO:teuthology.orchestra.run.smithi164.stderr: -- replaced with /var/log/ceph/cluster1-client.admin.41419.log.gz/var/log/ceph/cluster1-client.mirror.37399.log: 2024-09-14T09:12:13.292 INFO:teuthology.orchestra.run.smithi164.stderr: 55.8% -- replaced with /var/log/ceph/cluster1-client.mirror.37399.log.gz 2024-09-14T09:12:13.292 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56485.log 2024-09-14T09:12:13.292 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.58660.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67337.log 2024-09-14T09:12:13.292 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.56485.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.58660.log.gz 2024-09-14T09:12:13.292 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71436.log 2024-09-14T09:12:13.292 INFO:teuthology.orchestra.run.smithi164.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.56485.log.gzgzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.39102.log 2024-09-14T09:12:13.292 INFO:teuthology.orchestra.run.smithi164.stderr: 2024-09-14T09:12:13.293 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.67337.log: /var/log/ceph/cluster2-client.mirror.71436.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57965.log 2024-09-14T09:12:13.293 INFO:teuthology.orchestra.run.smithi164.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.67337.log.gz 2024-09-14T09:12:13.293 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.39102.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.71436.log.gz 2024-09-14T09:12:13.293 INFO:teuthology.orchestra.run.smithi164.stderr: 87.2% -- replaced with /var/log/ceph/cluster1-client.mirror.39102.log.gz 2024-09-14T09:12:13.293 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73517.log 2024-09-14T09:12:13.293 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.57965.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60571.log 2024-09-14T09:12:13.293 INFO:teuthology.orchestra.run.smithi164.stderr: 87.6%/var/log/ceph/cluster2-client.mirror.73517.log: -- replaced with /var/log/ceph/cluster1-client.mirror.57965.log.gz 2024-09-14T09:12:13.293 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59179.log 2024-09-14T09:12:13.293 INFO:teuthology.orchestra.run.smithi164.stderr: 93.5% 87.1% -- replaced with /var/log/ceph/cluster2-client.mirror.73517.log.gz 2024-09-14T09:12:13.293 INFO:teuthology.orchestra.run.smithi164.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.44609.log.gz 2024-09-14T09:12:13.294 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72998.log 2024-09-14T09:12:13.294 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.60571.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44437.log 2024-09-14T09:12:13.294 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.59179.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.60571.log.gz 2024-09-14T09:12:13.294 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.72998.log: 87.4%gzip -- replaced with /var/log/ceph/cluster1-client.mirror.59179.log.gz -5 2024-09-14T09:12:13.294 INFO:teuthology.orchestra.run.smithi164.stderr: --verbose -- /var/log/ceph/cluster2-client.mirror.48193.log 2024-09-14T09:12:13.294 INFO:teuthology.orchestra.run.smithi164.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.72998.log.gz 2024-09-14T09:12:13.294 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67787.log 2024-09-14T09:12:13.294 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.44437.log: /var/log/ceph/cluster2-client.mirror.48193.log: gzip -5 --verbose -- 79.6% /var/log/ceph/cluster1-client.mirror.40639.log 2024-09-14T09:12:13.294 INFO:teuthology.orchestra.run.smithi164.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.48193.log.gz 2024-09-14T09:12:13.294 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.67787.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.36254.log 2024-09-14T09:12:13.294 INFO:teuthology.orchestra.run.smithi164.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.67787.log.gz 2024-09-14T09:12:13.294 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37055.log 2024-09-14T09:12:13.294 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.40639.log: /var/log/ceph/cluster2-client.admin.36254.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.64274.log 2024-09-14T09:12:13.294 INFO:teuthology.orchestra.run.smithi164.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.36254.log.gz 2024-09-14T09:12:13.295 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.37055.log: 88.0% -- replaced with /var/log/ceph/cluster1-client.mirror.40639.log.gz 2024-09-14T09:12:13.295 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48297.log 2024-09-14T09:12:13.295 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose --/var/log/ceph/cluster1-client.mirror.3.64274.log: /var/log/ceph/cluster2-client.mirror.69814.log 2024-09-14T09:12:13.295 INFO:teuthology.orchestra.run.smithi164.stderr: 94.5% -- replaced with /var/log/ceph/cluster1-client.mirror.37055.log.gz 2024-09-14T09:12:13.295 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.48297.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64847.log 2024-09-14T09:12:13.295 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.69814.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.69814.log.gz 2024-09-14T09:12:13.295 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47527.log 2024-09-14T09:12:13.295 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.64847.log: 94.9% -- replaced with /var/log/ceph/cluster1-client.mirror.48297.log.gz 2024-09-14T09:12:13.295 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.62624.log 2024-09-14T09:12:13.295 INFO:teuthology.orchestra.run.smithi164.stderr: 93.4% -- replaced with /var/log/ceph/cluster2-client.mirror.44437.log.gz 2024-09-14T09:12:13.295 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.47527.log: 79.7% -- replaced with /var/log/ceph/cluster2-client.mirror.47527.log.gz 2024-09-14T09:12:13.295 INFO:teuthology.orchestra.run.smithi164.stderr: 90.1% -- replaced with /var/log/ceph/cluster2-client.mirror.64847.log.gz 2024-09-14T09:12:13.295 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54452.log 2024-09-14T09:12:13.295 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.62624.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65095.log 2024-09-14T09:12:13.295 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.54452.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.62624.log.gz 2024-09-14T09:12:13.296 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.68796.log 2024-09-14T09:12:13.296 INFO:teuthology.orchestra.run.smithi164.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.54452.log.gz 2024-09-14T09:12:13.296 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43566.log 2024-09-14T09:12:13.296 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.65095.log: /var/log/ceph/cluster1-client.mirror.0.68796.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44812.log 2024-09-14T09:12:13.296 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.43566.log: 90.1% -- replaced with /var/log/ceph/cluster2-client.mirror.65095.log.gz 2024-09-14T09:12:13.296 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46772.log 2024-09-14T09:12:13.296 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.44812.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.44812.log.gz 2024-09-14T09:12:13.296 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66191.log 2024-09-14T09:12:13.296 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.46772.log: 79.5% -- replaced with /var/log/ceph/cluster2-client.mirror.46772.log.gz 2024-09-14T09:12:13.296 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63534.log 2024-09-14T09:12:13.296 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.66191.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.66191.log.gz 2024-09-14T09:12:13.296 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64889.log 2024-09-14T09:12:13.325 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.63534.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.63534.log.gz 2024-09-14T09:12:13.325 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53268.log 2024-09-14T09:12:13.325 INFO:teuthology.orchestra.run.smithi164.stderr: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.43566.log.gz 2024-09-14T09:12:13.325 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.64889.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45754.log 2024-09-14T09:12:13.325 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.53268.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.64889.log.gz 2024-09-14T09:12:13.325 INFO:teuthology.orchestra.run.smithi164.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.53268.log.gz 2024-09-14T09:12:13.325 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.35708.log 2024-09-14T09:12:13.325 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47803.log 2024-09-14T09:12:13.325 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.admin.35708.log: /var/log/ceph/cluster2-client.mirror.45754.log: 93.2% 0.0% -- replaced with /var/log/ceph/cluster1-client.mirror.0.68796.log.gz 2024-09-14T09:12:13.325 INFO:teuthology.orchestra.run.smithi164.stderr: -- replaced with /var/log/ceph/cluster2-client.admin.35708.log.gz 79.8% -- replaced with /var/log/ceph/cluster2-client.mirror.45754.log.gz 2024-09-14T09:12:13.325 INFO:teuthology.orchestra.run.smithi164.stderr: 2024-09-14T09:12:13.326 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57563.log 2024-09-14T09:12:13.326 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.47803.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51810.log 2024-09-14T09:12:13.326 INFO:teuthology.orchestra.run.smithi164.stderr: 87.6%/var/log/ceph/cluster1-client.mirror.57563.log: -- replaced with /var/log/ceph/cluster2-client.mirror.47803.log.gz 2024-09-14T09:12:13.326 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.36928.log 2024-09-14T09:12:13.326 INFO:teuthology.orchestra.run.smithi164.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.57563.log.gz 2024-09-14T09:12:13.326 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59936.log 2024-09-14T09:12:13.326 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.51810.log: /var/log/ceph/cluster1-client.mirror.3.36928.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46432.log 2024-09-14T09:12:13.326 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.59936.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.51810.log.gz 2024-09-14T09:12:13.326 INFO:teuthology.orchestra.run.smithi164.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.59936.log.gz 2024-09-14T09:12:13.326 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60201.log 2024-09-14T09:12:13.326 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.37587.log 2024-09-14T09:12:13.326 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.46432.log: /var/log/ceph/cluster1-client.mirror.60201.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.46432.log.gz 2024-09-14T09:12:13.326 INFO:teuthology.orchestra.run.smithi164.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.60201.log.gz 2024-09-14T09:12:13.327 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.32854.log 2024-09-14T09:12:13.327 INFO:teuthology.orchestra.run.smithi164.stderr: 92.2% -- replaced with /var/log/ceph/cluster1-client.mirror.3.36928.log.gz/var/log/ceph/cluster2-client.admin.37587.log: 2024-09-14T09:12:13.327 INFO:teuthology.orchestra.run.smithi164.stderr: 80.0%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43292.log 2024-09-14T09:12:13.327 INFO:teuthology.orchestra.run.smithi164.stderr: -- replaced with /var/log/ceph/cluster2-client.admin.37587.log.gz 2024-09-14T09:12:13.327 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.admin.32854.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72602.log 2024-09-14T09:12:13.327 INFO:teuthology.orchestra.run.smithi164.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.32854.log.gz 2024-09-14T09:12:13.327 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.43292.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37712.log 2024-09-14T09:12:13.327 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.72602.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47745.log 2024-09-14T09:12:13.327 INFO:teuthology.orchestra.run.smithi164.stderr: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.43292.log.gz 2024-09-14T09:12:13.327 INFO:teuthology.orchestra.run.smithi164.stderr: 87.0% -- replaced with /var/log/ceph/cluster2-client.mirror.72602.log.gz 2024-09-14T09:12:13.327 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.37712.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.73390.log 2024-09-14T09:12:13.327 INFO:teuthology.orchestra.run.smithi164.stderr: 57.4% -- replaced with /var/log/ceph/cluster2-client.mirror.37712.log.gz 2024-09-14T09:12:13.327 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5/var/log/ceph/cluster2-client.mirror.47745.log: --verbose -- /var/log/ceph/cluster2-client.mirror.71271.log 2024-09-14T09:12:13.328 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.73390.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42613.log 2024-09-14T09:12:13.328 INFO:teuthology.orchestra.run.smithi164.stderr: 91.8% -- replaced with /var/log/ceph/cluster1-client.mirror.3.64274.log.gz 2024-09-14T09:12:13.328 INFO:teuthology.orchestra.run.smithi164.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.73390.log.gz 2024-09-14T09:12:13.328 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.71271.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72788.log 2024-09-14T09:12:13.328 INFO:teuthology.orchestra.run.smithi164.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.71271.log.gz 2024-09-14T09:12:13.328 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.42613.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.73413.log 2024-09-14T09:12:13.328 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.72788.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.35792.log 2024-09-14T09:12:13.328 INFO:teuthology.orchestra.run.smithi164.stderr: 79.7% -- replaced with /var/log/ceph/cluster2-client.mirror.72788.log.gz 2024-09-14T09:12:13.328 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.73413.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.73413.log.gz 2024-09-14T09:12:13.328 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64364.log 2024-09-14T09:12:13.328 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.admin.35792.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.35792.log.gzgzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51675.log 2024-09-14T09:12:13.328 INFO:teuthology.orchestra.run.smithi164.stderr: 2024-09-14T09:12:13.328 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62114.log 2024-09-14T09:12:13.329 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.64364.log: /var/log/ceph/cluster1-client.mirror.51675.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.64364.log.gz 2024-09-14T09:12:13.329 INFO:teuthology.orchestra.run.smithi164.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.51675.log.gz 2024-09-14T09:12:13.329 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45522.log 2024-09-14T09:12:13.329 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.62114.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.62114.log.gz 2024-09-14T09:12:13.329 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.71457.log 2024-09-14T09:12:13.329 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46188.log 2024-09-14T09:12:13.329 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.45522.log: /var/log/ceph/cluster1-client.mirror.71457.log: 93.4% -- replaced with /var/log/ceph/cluster2-client.mirror.42613.log.gz 2024-09-14T09:12:13.329 INFO:teuthology.orchestra.run.smithi164.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.71457.log.gz 2024-09-14T09:12:13.329 INFO:teuthology.orchestra.run.smithi164.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.45522.log.gz 2024-09-14T09:12:13.329 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43586.log 2024-09-14T09:12:13.329 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68909.log 2024-09-14T09:12:13.329 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.46188.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.35544.log 2024-09-14T09:12:13.329 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.43586.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.46188.log.gz 2024-09-14T09:12:13.329 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.68909.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55111.log 2024-09-14T09:12:13.330 INFO:teuthology.orchestra.run.smithi164.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.68909.log.gz 2024-09-14T09:12:13.330 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.admin.35544.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.71415.log 2024-09-14T09:12:13.330 INFO:teuthology.orchestra.run.smithi164.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.35544.log.gz 2024-09-14T09:12:13.330 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.55111.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68603.log 2024-09-14T09:12:13.330 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.71415.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.55111.log.gz 2024-09-14T09:12:13.330 INFO:teuthology.orchestra.run.smithi164.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.71415.log.gz 2024-09-14T09:12:13.330 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59107.log 2024-09-14T09:12:13.330 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5/var/log/ceph/cluster2-client.mirror.68603.log: --verbose -- /var/log/ceph/cluster1-client.mirror.72476.log 2024-09-14T09:12:13.330 INFO:teuthology.orchestra.run.smithi164.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.68603.log.gz 2024-09-14T09:12:13.330 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.59107.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47943.log 2024-09-14T09:12:13.330 INFO:teuthology.orchestra.run.smithi164.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.59107.log.gz 2024-09-14T09:12:13.330 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.72476.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66977.log 2024-09-14T09:12:13.330 INFO:teuthology.orchestra.run.smithi164.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.72476.log.gz 2024-09-14T09:12:13.331 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.47943.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50844.log 2024-09-14T09:12:13.331 INFO:teuthology.orchestra.run.smithi164.stderr: 93.4% -- replaced with /var/log/ceph/cluster1-client.mirror.43586.log.gz 2024-09-14T09:12:13.331 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.66977.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.47943.log.gz 2024-09-14T09:12:13.331 INFO:teuthology.orchestra.run.smithi164.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.66977.log.gz 2024-09-14T09:12:13.331 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66392.log 2024-09-14T09:12:13.331 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.50844.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52397.log 2024-09-14T09:12:13.331 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.66392.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41112.log 2024-09-14T09:12:13.331 INFO:teuthology.orchestra.run.smithi164.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.66392.log.gz 2024-09-14T09:12:13.331 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.52397.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65819.log 2024-09-14T09:12:13.331 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.41112.log: 92.4% -- replaced with /var/log/ceph/cluster2-client.mirror.50844.log.gz 2024-09-14T09:12:13.331 INFO:teuthology.orchestra.run.smithi164.stderr: 88.6% -- replaced with /var/log/ceph/cluster1-client.mirror.41112.log.gz 2024-09-14T09:12:13.331 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39817.log 2024-09-14T09:12:13.332 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.65819.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69147.log 2024-09-14T09:12:13.332 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.39817.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.65819.log.gz 2024-09-14T09:12:13.332 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.42379.log 2024-09-14T09:12:13.332 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.69147.log: 92.3% -- replaced with /var/log/ceph/cluster2-client.mirror.52397.log.gz 2024-09-14T09:12:13.332 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.62949.log 2024-09-14T09:12:13.332 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.admin.42379.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.42379.log.gz 2024-09-14T09:12:13.332 INFO:teuthology.orchestra.run.smithi164.stderr: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.69147.log.gz 2024-09-14T09:12:13.332 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45584.log 2024-09-14T09:12:13.332 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55553.log 2024-09-14T09:12:13.332 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.62949.log: /var/log/ceph/cluster2-client.mirror.45584.log: 79.7% 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.62949.log.gz 2024-09-14T09:12:13.332 INFO:teuthology.orchestra.run.smithi164.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.45584.log.gz 2024-09-14T09:12:13.332 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62351.log 2024-09-14T09:12:13.332 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67179.log 2024-09-14T09:12:13.332 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.55553.log: /var/log/ceph/cluster2-client.mirror.62351.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.55553.log.gz 2024-09-14T09:12:13.333 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73245.log 2024-09-14T09:12:13.333 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.67179.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.67179.log.gz 2024-09-14T09:12:13.333 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37027.log 2024-09-14T09:12:13.333 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.73245.log: 79.8% -- replaced with /var/log/ceph/cluster2-client.mirror.73245.log.gz 2024-09-14T09:12:13.333 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.62672.log 2024-09-14T09:12:13.336 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.37027.log: 95.0% -- replaced with /var/log/ceph/cluster1-client.mirror.37027.log.gz 2024-09-14T09:12:13.336 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65736.log 2024-09-14T09:12:13.336 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.62672.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.62672.log.gz 2024-09-14T09:12:13.336 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60937.log 2024-09-14T09:12:13.336 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.65736.log: 92.8% 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.65736.log.gz 2024-09-14T09:12:13.336 INFO:teuthology.orchestra.run.smithi164.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.62351.log.gz 2024-09-14T09:12:13.336 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67517.log 2024-09-14T09:12:13.336 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.74530.log 2024-09-14T09:12:13.336 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.60937.log: /var/log/ceph/cluster1-client.mirror.67517.log: 86.9% 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.60937.log.gz 2024-09-14T09:12:13.337 INFO:teuthology.orchestra.run.smithi164.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.67517.log.gz 2024-09-14T09:12:13.337 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42147.log 2024-09-14T09:12:13.337 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55180.log 2024-09-14T09:12:13.337 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.admin.74530.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.74530.log.gz 2024-09-14T09:12:13.337 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.42147.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45870.log 2024-09-14T09:12:13.338 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.55180.log: 92.3% -- replaced with /var/log/ceph/cluster2-client.mirror.55180.log.gz 2024-09-14T09:12:13.338 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53224.log 2024-09-14T09:12:13.338 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.45870.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.45870.log.gz 2024-09-14T09:12:13.338 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55356.log 2024-09-14T09:12:13.339 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.53224.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.53224.log.gz 2024-09-14T09:12:13.339 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.73866.log 2024-09-14T09:12:13.340 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.55356.log: 93.4% -- replaced with /var/log/ceph/cluster2-client.mirror.42147.log.gz 2024-09-14T09:12:13.344 INFO:teuthology.orchestra.run.smithi164.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.55356.log.gz 2024-09-14T09:12:13.344 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60105.log 2024-09-14T09:12:13.344 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.73866.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.mirror.73866.log.gz 2024-09-14T09:12:13.344 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70520.log 2024-09-14T09:12:13.345 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose/var/log/ceph/cluster1-client.mirror.60105.log: -- /var/log/ceph/cluster2-client.mirror.71999.log 2024-09-14T09:12:13.345 INFO:teuthology.orchestra.run.smithi164.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.60105.log.gz 2024-09-14T09:12:13.345 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.70520.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.70520.log.gz 2024-09-14T09:12:13.345 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58929.log 2024-09-14T09:12:13.345 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.71999.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41873.log 2024-09-14T09:12:13.345 INFO:teuthology.orchestra.run.smithi164.stderr: 86.9% -- replaced with /var/log/ceph/cluster2-client.mirror.71999.log.gz 2024-09-14T09:12:13.345 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.58929.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47825.log 2024-09-14T09:12:13.345 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.41873.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.58929.log.gz 2024-09-14T09:12:13.345 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48252.log 2024-09-14T09:12:13.345 INFO:teuthology.orchestra.run.smithi164.stderr: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.41873.log.gz/var/log/ceph/cluster2-client.mirror.47825.log: 2024-09-14T09:12:13.346 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45183.log 2024-09-14T09:12:13.363 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.48252.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69978.log 2024-09-14T09:12:13.364 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.45183.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.45183.log.gz 2024-09-14T09:12:13.365 INFO:teuthology.orchestra.run.smithi164.stderr: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.45971.log.gz 2024-09-14T09:12:13.375 INFO:teuthology.orchestra.run.smithi164.stderr: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.72390.log.gz 2024-09-14T09:12:13.376 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46727.log 2024-09-14T09:12:13.377 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.69978.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39753.log 2024-09-14T09:12:13.378 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.46727.log: 86.9% -- replaced with /var/log/ceph/cluster2-client.mirror.69978.log.gz 2024-09-14T09:12:13.378 INFO:teuthology.orchestra.run.smithi164.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.46727.log.gz 2024-09-14T09:12:13.378 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36861.log 2024-09-14T09:12:13.378 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.39753.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68886.log 2024-09-14T09:12:13.384 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.36861.log: 93.3% -- replaced with /var/log/ceph/cluster2-client.mirror.39753.log.gz 2024-09-14T09:12:13.384 INFO:teuthology.orchestra.run.smithi164.stderr: 93.9%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73475.log 2024-09-14T09:12:13.384 INFO:teuthology.orchestra.run.smithi164.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.36861.log.gz 2024-09-14T09:12:13.384 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.68886.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41491.log 2024-09-14T09:12:13.385 INFO:teuthology.orchestra.run.smithi164.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.68886.log.gz 2024-09-14T09:12:13.386 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.73475.log: 87.0% -- replaced with /var/log/ceph/cluster2-client.mirror.73475.log.gz 2024-09-14T09:12:13.386 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56743.log 2024-09-14T09:12:13.386 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose --/var/log/ceph/cluster2-client.mirror.41491.log: /var/log/ceph/cluster1-client.mirror.51241.log 2024-09-14T09:12:13.388 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.56743.log: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.41491.log.gz 2024-09-14T09:12:13.388 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43430.log 2024-09-14T09:12:13.388 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.51241.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.51241.log.gz 2024-09-14T09:12:13.389 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.56743.log.gz 2024-09-14T09:12:13.389 INFO:teuthology.orchestra.run.smithi164.stderr: -- /var/log/ceph/cluster1-client.mirror.43812.log 2024-09-14T09:12:13.389 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71584.log 2024-09-14T09:12:13.391 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.43430.log: /var/log/ceph/cluster1-client.mirror.43812.log: 89.5% -- replaced with /var/log/ceph/cluster1-client.mirror.43430.log.gz 2024-09-14T09:12:13.391 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71291.log 2024-09-14T09:12:13.391 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.71584.log: 86.9% -- replaced with /var/log/ceph/cluster2-client.mirror.71584.log.gz 2024-09-14T09:12:13.391 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63885.log 2024-09-14T09:12:13.392 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.71291.log: 90.3% -- replaced with /var/log/ceph/cluster2-client.mirror.71291.log.gz 2024-09-14T09:12:13.394 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69271.log 2024-09-14T09:12:13.394 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.63885.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.63885.log.gz 2024-09-14T09:12:13.394 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42796.log 2024-09-14T09:12:13.394 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.69271.log: 87.2% -- replaced with /var/log/ceph/cluster1-client.mirror.69271.log.gz 2024-09-14T09:12:13.397 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54109.log 2024-09-14T09:12:13.397 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.42796.log: 93.3% -- replaced with /var/log/ceph/cluster1-client.mirror.43812.log.gz 2024-09-14T09:12:13.397 INFO:teuthology.orchestra.run.smithi164.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.42796.log.gz 2024-09-14T09:12:13.397 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61186.log 2024-09-14T09:12:13.397 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose/var/log/ceph/cluster2-client.mirror.54109.log: -- /var/log/ceph/cluster2-client.mirror.41157.log 2024-09-14T09:12:13.397 INFO:teuthology.orchestra.run.smithi164.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.54109.log.gz 2024-09-14T09:12:13.397 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.61186.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69521.log 2024-09-14T09:12:13.398 INFO:teuthology.orchestra.run.smithi164.stderr: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.61186.log.gz 2024-09-14T09:12:13.398 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.41157.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49410.log 2024-09-14T09:12:13.398 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.69521.log: 87.0% -- replaced with /var/log/ceph/cluster2-client.mirror.69521.log.gz 2024-09-14T09:12:13.398 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45621.log 2024-09-14T09:12:13.399 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.49410.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.49410.log.gz 2024-09-14T09:12:13.402 INFO:teuthology.orchestra.run.smithi164.stderr: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.46474.log.gz 2024-09-14T09:12:13.402 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.40403.log 2024-09-14T09:12:13.402 INFO:teuthology.orchestra.run.smithi164.stderr:gzip/var/log/ceph/cluster2-client.mirror.45621.log: -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68734.log 2024-09-14T09:12:13.402 INFO:teuthology.orchestra.run.smithi164.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.45621.log.gz 2024-09-14T09:12:13.402 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.40403.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67674.log 2024-09-14T09:12:13.402 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.68734.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.68734.log.gz 2024-09-14T09:12:13.402 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69562.log 2024-09-14T09:12:13.404 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.67674.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.67674.log.gz 2024-09-14T09:12:13.405 INFO:teuthology.orchestra.run.smithi164.stderr: 93.4% -- replaced with /var/log/ceph/cluster2-client.mirror.41157.log.gz 2024-09-14T09:12:13.405 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64447.log 2024-09-14T09:12:13.405 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.69562.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.37621.log 2024-09-14T09:12:13.405 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.64447.log: 86.9% -- replaced with /var/log/ceph/cluster2-client.mirror.69562.log.gz 2024-09-14T09:12:13.405 INFO:teuthology.orchestra.run.smithi164.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.64447.log.gz 2024-09-14T09:12:13.405 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71786.log 2024-09-14T09:12:13.405 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52625.log 2024-09-14T09:12:13.406 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.0.37621.log: /var/log/ceph/cluster2-client.mirror.71786.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.71786.log.gz 2024-09-14T09:12:13.406 INFO:teuthology.orchestra.run.smithi164.stderr: 93.4% -- replaced with /var/log/ceph/cluster1-client.mirror.40403.log.gz 2024-09-14T09:12:13.406 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56301.log 2024-09-14T09:12:13.407 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.52625.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60667.log 2024-09-14T09:12:13.408 INFO:teuthology.orchestra.run.smithi164.stderr: 92.7% -- replaced with /var/log/ceph/cluster2-client.mirror.0.37621.log.gz 2024-09-14T09:12:13.408 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.56301.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.52625.log.gz 2024-09-14T09:12:13.408 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45279.log 2024-09-14T09:12:13.409 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46970.log 2024-09-14T09:12:13.409 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.60667.log: /var/log/ceph/cluster2-client.mirror.45279.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.60667.log.gz 2024-09-14T09:12:13.409 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50189.log 2024-09-14T09:12:13.409 INFO:teuthology.orchestra.run.smithi164.stderr: 92.2% -- replaced with /var/log/ceph/cluster1-client.mirror.56301.log.gz 2024-09-14T09:12:13.412 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.46970.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.46970.log.gz 2024-09-14T09:12:13.412 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50450.log 2024-09-14T09:12:13.412 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.50189.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40775.log 2024-09-14T09:12:13.412 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.50450.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.50189.log.gz 2024-09-14T09:12:13.412 INFO:teuthology.orchestra.run.smithi164.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.50450.log.gz 2024-09-14T09:12:13.412 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41302.log 2024-09-14T09:12:13.412 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.40775.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51422.log 2024-09-14T09:12:13.417 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.41302.log: 93.4% -- replaced with /var/log/ceph/cluster2-client.mirror.40775.log.gz 2024-09-14T09:12:13.418 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.52053.log 2024-09-14T09:12:13.419 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.51422.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.51422.log.gz 2024-09-14T09:12:13.419 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38820.log 2024-09-14T09:12:13.419 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.0.52053.log: 93.9% -- replaced with /var/log/ceph/cluster2-client.mirror.45279.log.gz 2024-09-14T09:12:13.420 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63348.log 2024-09-14T09:12:13.421 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.38820.log: 73.8% -- replaced with /var/log/ceph/cluster2-client.mirror.38820.log.gz 2024-09-14T09:12:13.422 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52929.log 2024-09-14T09:12:13.423 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.63348.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.63348.log.gz 2024-09-14T09:12:13.423 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68583.log 2024-09-14T09:12:13.424 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.52929.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.52929.log.gz 2024-09-14T09:12:13.424 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.62901.log 2024-09-14T09:12:13.425 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.68583.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.68583.log.gz 2024-09-14T09:12:13.426 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61389.log 2024-09-14T09:12:13.426 INFO:teuthology.orchestra.run.smithi135.stderr: 90.2% -- replaced with /var/log/ceph/cluster1-mon.a.log.gz 2024-09-14T09:12:13.427 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.62901.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.62901.log.gz 2024-09-14T09:12:13.428 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.39150.log 2024-09-14T09:12:13.428 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.61389.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.61389.log.gz 2024-09-14T09:12:13.429 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63203.log 2024-09-14T09:12:13.429 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.39150.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.39150.log.gz 2024-09-14T09:12:13.430 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52485.log 2024-09-14T09:12:13.430 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.63203.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.63203.log.gz 2024-09-14T09:12:13.431 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63286.log 2024-09-14T09:12:13.432 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.52485.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.52485.log.gz 2024-09-14T09:12:13.433 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57720.log 2024-09-14T09:12:13.435 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.63286.log: 93.4% -- replaced with /var/log/ceph/cluster1-client.mirror.0.52053.log.gz 2024-09-14T09:12:13.435 INFO:teuthology.orchestra.run.smithi164.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.63286.log.gz 2024-09-14T09:12:13.435 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70996.log 2024-09-14T09:12:13.435 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.57720.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64529.log 2024-09-14T09:12:13.435 INFO:teuthology.orchestra.run.smithi164.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.57720.log.gz 2024-09-14T09:12:13.435 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.70996.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53182.log 2024-09-14T09:12:13.435 INFO:teuthology.orchestra.run.smithi164.stderr: 87.0% -- replaced with /var/log/ceph/cluster2-client.mirror.70996.log.gz 2024-09-14T09:12:13.436 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.64529.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68147.log 2024-09-14T09:12:13.436 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.53182.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.53182.log.gz 2024-09-14T09:12:13.439 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65943.log 2024-09-14T09:12:13.439 INFO:teuthology.orchestra.run.smithi164.stderr: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.64529.log.gz 2024-09-14T09:12:13.439 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.68147.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64682.log 2024-09-14T09:12:13.439 INFO:teuthology.orchestra.run.smithi164.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.68147.log.gz 2024-09-14T09:12:13.439 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.65943.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44050.log 2024-09-14T09:12:13.439 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.64682.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.65943.log.gz 2024-09-14T09:12:13.439 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43064.log 2024-09-14T09:12:13.439 INFO:teuthology.orchestra.run.smithi164.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.64682.log.gz 2024-09-14T09:12:13.439 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.44050.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55995.log 2024-09-14T09:12:13.440 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.43064.log: 89.0% -- replaced with /var/log/ceph/cluster1-client.mirror.43064.log.gz 2024-09-14T09:12:13.440 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.1.37002.log 2024-09-14T09:12:13.440 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.55995.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.55995.log.gz 2024-09-14T09:12:13.441 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63244.log 2024-09-14T09:12:13.445 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.1.37002.log: 93.4% -- replaced with /var/log/ceph/cluster2-client.mirror.44050.log.gz 2024-09-14T09:12:13.445 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69896.log 2024-09-14T09:12:13.447 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.63244.log: 90.1% -- replaced with /var/log/ceph/cluster2-client.mirror.63244.log.gz 2024-09-14T09:12:13.447 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57433.log 2024-09-14T09:12:13.448 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.69896.log: 86.9% -- replaced with /var/log/ceph/cluster2-client.mirror.69896.log.gz 2024-09-14T09:12:13.448 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58419.log 2024-09-14T09:12:13.449 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.57433.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.57433.log.gz 2024-09-14T09:12:13.449 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58953.log 2024-09-14T09:12:13.450 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.58419.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.58419.log.gz 2024-09-14T09:12:13.451 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66459.log 2024-09-14T09:12:13.452 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.58953.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.58953.log.gz 2024-09-14T09:12:13.452 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44912.log 2024-09-14T09:12:13.452 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.66459.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.66459.log.gz 2024-09-14T09:12:13.453 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44772.log 2024-09-14T09:12:13.454 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.44912.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.44912.log.gz 2024-09-14T09:12:13.454 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.39773.log 2024-09-14T09:12:13.456 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.44772.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.44772.log.gz 2024-09-14T09:12:13.456 INFO:teuthology.orchestra.run.smithi164.stderr: 92.4%gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50570.log 2024-09-14T09:12:13.456 INFO:teuthology.orchestra.run.smithi164.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.1.37002.log.gz 2024-09-14T09:12:13.456 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.39773.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69501.log 2024-09-14T09:12:13.456 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.50570.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.50570.log.gz 2024-09-14T09:12:13.457 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.73325.log 2024-09-14T09:12:13.457 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.69501.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.69501.log.gz 2024-09-14T09:12:13.458 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50120.log 2024-09-14T09:12:13.458 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.73325.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.73325.log.gz 2024-09-14T09:12:13.459 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61716.log 2024-09-14T09:12:13.460 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.50120.log: 1.3% -- replaced with /var/log/ceph/cluster1-client.mirror.50120.log.gz 2024-09-14T09:12:13.460 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67877.log 2024-09-14T09:12:13.461 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.61716.log: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.48525.log.gz 2024-09-14T09:12:13.461 INFO:teuthology.orchestra.run.smithi164.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.61716.log.gz 2024-09-14T09:12:13.491 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68282.log 2024-09-14T09:12:13.491 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5/var/log/ceph/cluster1-client.mirror.67877.log: --verbose 93.3% -- replaced with /var/log/ceph/cluster1-client.mirror.39773.log.gz 2024-09-14T09:12:13.492 INFO:teuthology.orchestra.run.smithi164.stderr: -- /var/log/ceph/cluster2-client.mirror.46890.log 2024-09-14T09:12:13.492 INFO:teuthology.orchestra.run.smithi164.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.67877.log.gz 2024-09-14T09:12:13.492 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose --/var/log/ceph/cluster1-client.mirror.68282.log: /var/log/ceph/cluster1-client.mirror.58773.log 2024-09-14T09:12:13.492 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.46890.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.68282.log.gz 2024-09-14T09:12:13.492 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65798.log 2024-09-14T09:12:13.492 INFO:teuthology.orchestra.run.smithi164.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.46890.log.gz 2024-09-14T09:12:13.492 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.58773.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67112.log 2024-09-14T09:12:13.492 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.65798.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44832.log 2024-09-14T09:12:13.492 INFO:teuthology.orchestra.run.smithi164.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.65798.log.gz 2024-09-14T09:12:13.492 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.67112.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.67112.log.gz 2024-09-14T09:12:13.492 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60222.log 2024-09-14T09:12:13.492 INFO:teuthology.orchestra.run.smithi164.stderr: 92.2% -- replaced with /var/log/ceph/cluster1-client.mirror.58773.log.gz 2024-09-14T09:12:13.492 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.44832.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69705.log 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.44832.log.gz 2024-09-14T09:12:13.493 INFO:teuthology.orchestra.run.smithi164.stderr: 2024-09-14T09:12:13.493 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.60222.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60865.log 2024-09-14T09:12:13.493 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.69705.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53140.log 2024-09-14T09:12:13.493 INFO:teuthology.orchestra.run.smithi164.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.69705.log.gz 2024-09-14T09:12:13.493 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.60865.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66527.log 2024-09-14T09:12:13.493 INFO:teuthology.orchestra.run.smithi164.stderr: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.60222.log.gz 2024-09-14T09:12:13.493 INFO:teuthology.orchestra.run.smithi164.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.60865.log.gz 2024-09-14T09:12:13.493 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.53140.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70184.log 2024-09-14T09:12:13.493 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose/var/log/ceph/cluster1-client.mirror.66527.log: -- /var/log/ceph/cluster1-client.mirror.56883.log 2024-09-14T09:12:13.493 INFO:teuthology.orchestra.run.smithi164.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.66527.log.gz 2024-09-14T09:12:13.493 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.70184.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.74034.log 2024-09-14T09:12:13.493 INFO:teuthology.orchestra.run.smithi164.stderr: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.53140.log.gz 2024-09-14T09:12:13.493 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.56883.log: 87.3%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72747.log 2024-09-14T09:12:13.494 INFO:teuthology.orchestra.run.smithi164.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.70184.log.gz 2024-09-14T09:12:13.494 INFO:teuthology.orchestra.run.smithi164.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.56883.log.gz 2024-09-14T09:12:13.494 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.admin.74034.log: 0.0%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51173.log 2024-09-14T09:12:13.494 INFO:teuthology.orchestra.run.smithi164.stderr: -- replaced with /var/log/ceph/cluster2-client.admin.74034.log.gz 2024-09-14T09:12:13.494 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.72747.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.38667.log 2024-09-14T09:12:13.494 INFO:teuthology.orchestra.run.smithi164.stderr: 87.8%/var/log/ceph/cluster2-client.mirror.51173.log: -- replaced with /var/log/ceph/cluster2-client.mirror.72747.log.gz 2024-09-14T09:12:13.494 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50932.log 2024-09-14T09:12:13.494 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.admin.38667.log: 88.9%gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.40711.log 2024-09-14T09:12:13.494 INFO:teuthology.orchestra.run.smithi164.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.51173.log.gz 2024-09-14T09:12:13.494 INFO:teuthology.orchestra.run.smithi164.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.38667.log.gz 2024-09-14T09:12:13.494 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.50932.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53884.log 2024-09-14T09:12:13.494 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56506.log 2024-09-14T09:12:13.494 INFO:teuthology.orchestra.run.smithi164.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.50932.log.gz 2024-09-14T09:12:13.494 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.40711.log: /var/log/ceph/cluster1-client.mirror.53884.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63906.log 2024-09-14T09:12:13.495 INFO:teuthology.orchestra.run.smithi164.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.40711.log.gz 2024-09-14T09:12:13.495 INFO:teuthology.orchestra.run.smithi164.stderr: 87.4%/var/log/ceph/cluster2-client.mirror.56506.log: -- replaced with /var/log/ceph/cluster1-client.mirror.53884.log.gz 2024-09-14T09:12:13.495 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70602.log 2024-09-14T09:12:13.495 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68417.log 2024-09-14T09:12:13.495 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.63906.log: /var/log/ceph/cluster2-client.mirror.70602.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.63906.log.gz 2024-09-14T09:12:13.495 INFO:teuthology.orchestra.run.smithi164.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.70602.log.gz 2024-09-14T09:12:13.495 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61961.log 2024-09-14T09:12:13.495 INFO:teuthology.orchestra.run.smithi164.stderr: 92.2% -- replaced with /var/log/ceph/cluster2-client.mirror.56506.log.gz 2024-09-14T09:12:13.495 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5/var/log/ceph/cluster1-client.mirror.68417.log: --verbose -- /var/log/ceph/cluster1-client.mirror.55577.log 2024-09-14T09:12:13.495 INFO:teuthology.orchestra.run.smithi164.stderr: 86.9%/var/log/ceph/cluster1-client.mirror.61961.log: -- replaced with /var/log/ceph/cluster1-client.mirror.68417.log.gz 2024-09-14T09:12:13.495 INFO:teuthology.orchestra.run.smithi164.stderr: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.41302.log.gz 2024-09-14T09:12:13.495 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55664.log 2024-09-14T09:12:13.495 INFO:teuthology.orchestra.run.smithi164.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.61961.log.gz 2024-09-14T09:12:13.495 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.55577.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64599.log 2024-09-14T09:12:13.496 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.55664.log: 87.7%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54226.log 2024-09-14T09:12:13.496 INFO:teuthology.orchestra.run.smithi164.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.55577.log.gz 2024-09-14T09:12:13.496 INFO:teuthology.orchestra.run.smithi164.stderr:gzip/var/log/ceph/cluster2-client.mirror.64599.log: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48403.log 2024-09-14T09:12:13.496 INFO:teuthology.orchestra.run.smithi164.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.55664.log.gz 2024-09-14T09:12:13.496 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.54226.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53510.log 2024-09-14T09:12:13.496 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.48403.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51768.log 2024-09-14T09:12:13.496 INFO:teuthology.orchestra.run.smithi164.stderr: 90.1% -- replaced with /var/log/ceph/cluster2-client.mirror.64599.log.gz 2024-09-14T09:12:13.496 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.53510.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.48403.log.gz 2024-09-14T09:12:13.496 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69855.log 2024-09-14T09:12:13.496 INFO:teuthology.orchestra.run.smithi164.stderr: 92.4% -- replaced with /var/log/ceph/cluster2-client.mirror.54226.log.gz 2024-09-14T09:12:13.496 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.51768.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47643.log 2024-09-14T09:12:13.496 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.69855.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58124.log 2024-09-14T09:12:13.496 INFO:teuthology.orchestra.run.smithi164.stderr: 87.0% -- replaced with /var/log/ceph/cluster2-client.mirror.69855.log.gz 2024-09-14T09:12:13.497 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.47643.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49330.log 2024-09-14T09:12:13.497 INFO:teuthology.orchestra.run.smithi164.stderr: 87.5% 92.5%/var/log/ceph/cluster2-client.mirror.58124.log: -- replaced with /var/log/ceph/cluster2-client.mirror.51768.log.gz 2024-09-14T09:12:13.497 INFO:teuthology.orchestra.run.smithi164.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.47643.log.gz 2024-09-14T09:12:13.497 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose 88.3% -- /var/log/ceph/cluster1-client.admin.41820.log 2024-09-14T09:12:13.497 INFO:teuthology.orchestra.run.smithi164.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.58124.log.gz 2024-09-14T09:12:13.497 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.49330.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55822.log 2024-09-14T09:12:13.497 INFO:teuthology.orchestra.run.smithi164.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.49330.log.gz 2024-09-14T09:12:13.497 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.admin.41820.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70118.log 2024-09-14T09:12:13.497 INFO:teuthology.orchestra.run.smithi164.stderr: 87.9% -- replaced with /var/log/ceph/cluster1-client.admin.41820.log.gz/var/log/ceph/cluster1-client.mirror.55822.log: 2024-09-14T09:12:13.497 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52553.log 2024-09-14T09:12:13.497 INFO:teuthology.orchestra.run.smithi164.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.55822.log.gz/var/log/ceph/cluster2-client.mirror.70118.log: 2024-09-14T09:12:13.497 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55443.log 2024-09-14T09:12:13.497 INFO:teuthology.orchestra.run.smithi164.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.70118.log.gz 2024-09-14T09:12:13.497 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.52553.log: gzip -5 --verbose -- /var/log/ceph/cluster2-osd.2.log 2024-09-14T09:12:13.498 INFO:teuthology.orchestra.run.smithi164.stderr: 87.6%/var/log/ceph/cluster2-client.mirror.55443.log: -- replaced with /var/log/ceph/cluster1-client.mirror.52553.log.gz 2024-09-14T09:12:13.498 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66149.log 2024-09-14T09:12:13.498 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-osd.2.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.55443.log.gz 2024-09-14T09:12:13.498 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51723.log 2024-09-14T09:12:13.498 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52907.log 2024-09-14T09:12:13.498 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.66149.log: /var/log/ceph/cluster1-client.mirror.51723.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.51723.log.gz 2024-09-14T09:12:13.498 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60177.log 2024-09-14T09:12:13.498 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.52907.log: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.66149.log.gz 2024-09-14T09:12:13.498 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52529.log 2024-09-14T09:12:13.498 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.60177.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.60177.log.gz 2024-09-14T09:12:13.498 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67652.log 2024-09-14T09:12:13.498 INFO:teuthology.orchestra.run.smithi164.stderr: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.52907.log.gz 2024-09-14T09:12:13.498 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.52529.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69313.log 2024-09-14T09:12:13.498 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.67652.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.52529.log.gz 2024-09-14T09:12:13.499 INFO:teuthology.orchestra.run.smithi164.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.67652.log.gz 2024-09-14T09:12:13.499 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44500.log 2024-09-14T09:12:13.499 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.69313.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42833.log 2024-09-14T09:12:13.499 INFO:teuthology.orchestra.run.smithi164.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.69313.log.gz 2024-09-14T09:12:13.504 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.44500.log: 93.4% -- replaced with /var/log/ceph/cluster1-client.mirror.44500.log.gz 2024-09-14T09:12:13.504 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59309.log 2024-09-14T09:12:13.504 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57274.log 2024-09-14T09:12:13.504 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.42833.log: /var/log/ceph/cluster1-client.mirror.59309.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.59309.log.gz 93.9% 2024-09-14T09:12:13.504 INFO:teuthology.orchestra.run.smithi164.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.42833.log.gz 2024-09-14T09:12:13.504 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.38623.log 2024-09-14T09:12:13.504 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59131.log 2024-09-14T09:12:13.504 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.57274.log: /var/log/ceph/cluster1-client.admin.38623.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.57274.log.gz 2024-09-14T09:12:13.505 INFO:teuthology.orchestra.run.smithi164.stderr: 87.4%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51539.log 2024-09-14T09:12:13.505 INFO:teuthology.orchestra.run.smithi164.stderr: -- replaced with /var/log/ceph/cluster1-client.admin.38623.log.gz 2024-09-14T09:12:13.505 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.59131.log: 86.4% -- replaced with /var/log/ceph/cluster1-client.mirror.59131.log.gz 2024-09-14T09:12:13.505 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50305.log 2024-09-14T09:12:13.505 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70098.log 2024-09-14T09:12:13.505 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.51539.log: /var/log/ceph/cluster1-client.mirror.50305.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.50305.log.gz 2024-09-14T09:12:13.505 INFO:teuthology.orchestra.run.smithi164.stderr: 92.4% -- replaced with /var/log/ceph/cluster2-client.mirror.51539.log.gz 2024-09-14T09:12:13.505 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.62138.log 2024-09-14T09:12:13.505 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44892.log 2024-09-14T09:12:13.505 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.70098.log: /var/log/ceph/cluster1-client.mirror.62138.log: 87.7% 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.70098.log.gz 2024-09-14T09:12:13.505 INFO:teuthology.orchestra.run.smithi164.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.62138.log.gz 2024-09-14T09:12:13.505 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54701.log 2024-09-14T09:12:13.505 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64620.log 2024-09-14T09:12:13.505 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.44892.log: /var/log/ceph/cluster1-client.mirror.54701.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.44892.log.gz 2024-09-14T09:12:13.506 INFO:teuthology.orchestra.run.smithi164.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.54701.log.gz 2024-09-14T09:12:13.506 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49648.log 2024-09-14T09:12:13.506 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.64620.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.64620.log.gz 2024-09-14T09:12:13.506 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71769.log 2024-09-14T09:12:13.506 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64274.log 2024-09-14T09:12:13.508 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.49648.log: /var/log/ceph/cluster2-client.mirror.71769.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.49648.log.gz 2024-09-14T09:12:13.508 INFO:teuthology.orchestra.run.smithi164.stderr: 79.8% -- replaced with /var/log/ceph/cluster2-client.mirror.71769.log.gz 2024-09-14T09:12:13.508 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61227.log 2024-09-14T09:12:13.508 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51892.log 2024-09-14T09:12:13.508 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.64274.log: /var/log/ceph/cluster1-client.mirror.61227.log: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.64274.log.gz 2024-09-14T09:12:13.508 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58636.log 2024-09-14T09:12:13.508 INFO:teuthology.orchestra.run.smithi164.stderr: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.61227.log.gz 2024-09-14T09:12:13.508 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.51892.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.51892.log.gz 2024-09-14T09:12:13.509 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63802.log 2024-09-14T09:12:13.509 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44692.log 2024-09-14T09:12:13.509 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.58636.log: /var/log/ceph/cluster2-client.mirror.63802.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.58636.log.gz 2024-09-14T09:12:13.509 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.39371.log 2024-09-14T09:12:13.510 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.44692.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.44692.log.gz 2024-09-14T09:12:13.510 INFO:teuthology.orchestra.run.smithi164.stderr: 90.1% -- replaced with /var/log/ceph/cluster2-client.mirror.63802.log.gz 2024-09-14T09:12:13.510 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52646.log 2024-09-14T09:12:13.511 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72310.log 2024-09-14T09:12:13.511 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.admin.39371.log: /var/log/ceph/cluster2-client.mirror.52646.log: 87.1% -- replaced with /var/log/ceph/cluster1-client.admin.39371.log.gz 2024-09-14T09:12:13.512 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42104.log 2024-09-14T09:12:13.513 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.72310.log: 92.7% -- replaced with /var/log/ceph/cluster2-client.mirror.52646.log.gz 2024-09-14T09:12:13.513 INFO:teuthology.orchestra.run.smithi164.stderr: 90.3% -- replaced with /var/log/ceph/cluster2-client.mirror.72310.log.gz 2024-09-14T09:12:13.514 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.40795.log 2024-09-14T09:12:13.514 INFO:teuthology.orchestra.run.smithi164.stderr: 93.7%gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51000.log 2024-09-14T09:12:13.514 INFO:teuthology.orchestra.run.smithi164.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.53510.log.gz 2024-09-14T09:12:13.515 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.42104.log: /var/log/ceph/cluster1-client.mirror.40795.log: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.47745.log.gz 2024-09-14T09:12:13.518 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.37644.log 2024-09-14T09:12:13.518 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.51000.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64930.log 2024-09-14T09:12:13.518 INFO:teuthology.orchestra.run.smithi164.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.51000.log.gz 2024-09-14T09:12:13.518 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.admin.37644.log: 66.9% -- replaced with /var/log/ceph/cluster2-client.admin.37644.log.gz 2024-09-14T09:12:13.518 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.71914.log 2024-09-14T09:12:13.518 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.64930.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72725.log 2024-09-14T09:12:13.518 INFO:teuthology.orchestra.run.smithi164.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.64930.log.gz/var/log/ceph/cluster1-client.mirror.71914.log: 2024-09-14T09:12:13.519 INFO:teuthology.orchestra.run.smithi164.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.71914.log.gz 2024-09-14T09:12:13.519 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64092.log 2024-09-14T09:12:13.519 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73369.log 2024-09-14T09:12:13.534 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.72725.log: /var/log/ceph/cluster2-client.mirror.64092.log: 87.0% -- replaced with /var/log/ceph/cluster2-client.mirror.72725.log.gz 2024-09-14T09:12:13.534 INFO:teuthology.orchestra.run.smithi164.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.64092.log.gz 2024-09-14T09:12:13.534 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37822.log 2024-09-14T09:12:13.534 INFO:teuthology.orchestra.run.smithi164.stderr: 93.4% -- replaced with /var/log/ceph/cluster2-client.mirror.42104.log.gz 2024-09-14T09:12:13.534 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62048.log 2024-09-14T09:12:13.534 INFO:teuthology.orchestra.run.smithi164.stderr: 93.4% -- replaced with /var/log/ceph/cluster1-client.mirror.40795.log.gz 2024-09-14T09:12:13.534 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.73369.log: /var/log/ceph/cluster2-client.mirror.37822.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66211.log 2024-09-14T09:12:13.534 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.62048.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.37822.log.gz 2024-09-14T09:12:13.534 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53695.log 2024-09-14T09:12:13.534 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.66211.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43523.log 2024-09-14T09:12:13.534 INFO:teuthology.orchestra.run.smithi164.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.73369.log.gz 2024-09-14T09:12:13.534 INFO:teuthology.orchestra.run.smithi164.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.62048.log.gz 2024-09-14T09:12:13.534 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.53695.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71034.log 2024-09-14T09:12:13.535 INFO:teuthology.orchestra.run.smithi164.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.53695.log.gz/var/log/ceph/cluster2-client.mirror.43523.log: 2024-09-14T09:12:13.535 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52688.log 2024-09-14T09:12:13.535 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.71034.log: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.66211.log.gz 2024-09-14T09:12:13.535 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47066.log 2024-09-14T09:12:13.535 INFO:teuthology.orchestra.run.smithi164.stderr: 76.8% -- replaced with /var/log/ceph/cluster2-client.mirror.71034.log.gz 2024-09-14T09:12:13.535 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67697.log 2024-09-14T09:12:13.535 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.52688.log: /var/log/ceph/cluster2-client.mirror.47066.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66864.log 2024-09-14T09:12:13.535 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.67697.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.52688.log.gz 2024-09-14T09:12:13.535 INFO:teuthology.orchestra.run.smithi164.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.67697.log.gz 2024-09-14T09:12:13.535 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65033.log 2024-09-14T09:12:13.535 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48662.log 2024-09-14T09:12:13.535 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.66864.log: /var/log/ceph/cluster2-client.mirror.65033.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.66864.log.gz 2024-09-14T09:12:13.535 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61365.log 2024-09-14T09:12:13.536 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.48662.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.48662.log.gz 2024-09-14T09:12:13.536 INFO:teuthology.orchestra.run.smithi164.stderr: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.65033.log.gz 2024-09-14T09:12:13.536 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.40117.log 2024-09-14T09:12:13.536 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52794.log 2024-09-14T09:12:13.536 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.61365.log: /var/log/ceph/cluster1-client.mirror.40117.log: 93.4% 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.61365.log.gz 2024-09-14T09:12:13.536 INFO:teuthology.orchestra.run.smithi164.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.43523.log.gz 2024-09-14T09:12:13.536 INFO:teuthology.orchestra.run.smithi164.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.40117.log.gz 2024-09-14T09:12:13.536 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37695.log 2024-09-14T09:12:13.536 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose/var/log/ceph/cluster1-client.mirror.52794.log: -- /var/log/ceph/cluster1-client.mirror.59018.log 2024-09-14T09:12:13.536 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.37695.log: 87.3%gzip -5 --verbose -- 2.3% /var/log/ceph/cluster1-client.mirror.53647.log 2024-09-14T09:12:13.536 INFO:teuthology.orchestra.run.smithi164.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.37695.log.gz 2024-09-14T09:12:13.536 INFO:teuthology.orchestra.run.smithi164.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.52794.log.gz 2024-09-14T09:12:13.536 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose --/var/log/ceph/cluster1-client.mirror.59018.log: /var/log/ceph/cluster1-client.mirror.51699.log 2024-09-14T09:12:13.536 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.53647.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47214.log 2024-09-14T09:12:13.537 INFO:teuthology.orchestra.run.smithi164.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.53647.log.gz 2024-09-14T09:12:13.537 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.51699.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57587.log 2024-09-14T09:12:13.537 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.47214.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.51699.log.gz 2024-09-14T09:12:13.537 INFO:teuthology.orchestra.run.smithi164.stderr: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.59018.log.gz 2024-09-14T09:12:13.537 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40277.log 2024-09-14T09:12:13.537 INFO:teuthology.orchestra.run.smithi164.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.47214.log.gz 2024-09-14T09:12:13.537 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58327.log 2024-09-14T09:12:13.537 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.57587.log: /var/log/ceph/cluster2-client.mirror.40277.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49796.log 2024-09-14T09:12:13.537 INFO:teuthology.orchestra.run.smithi164.stderr: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.57587.log.gz 2024-09-14T09:12:13.537 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.58327.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48043.log 2024-09-14T09:12:13.537 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.49796.log: 79.7% -- replaced with /var/log/ceph/cluster2-client.mirror.49796.log.gz 2024-09-14T09:12:13.537 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66025.log 2024-09-14T09:12:13.537 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.48043.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.48043.log.gz 2024-09-14T09:12:13.537 INFO:teuthology.orchestra.run.smithi164.stderr: 92.3%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62759.log 2024-09-14T09:12:13.538 INFO:teuthology.orchestra.run.smithi164.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.58327.log.gz 2024-09-14T09:12:13.538 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.66025.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42568.log 2024-09-14T09:12:13.561 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.62759.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.62759.log.gz 2024-09-14T09:12:13.561 INFO:teuthology.orchestra.run.smithi164.stderr: 90.1% -- replaced with /var/log/ceph/cluster2-client.mirror.66025.log.gz 2024-09-14T09:12:13.561 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48820.log 2024-09-14T09:12:13.561 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55529.log 2024-09-14T09:12:13.561 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.48820.log: /var/log/ceph/cluster1-client.mirror.42568.log: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.40277.log.gz 2024-09-14T09:12:13.562 INFO:teuthology.orchestra.run.smithi164.stderr: 89.2% 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.48820.log.gz 2024-09-14T09:12:13.562 INFO:teuthology.orchestra.run.smithi164.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.42568.log.gz 2024-09-14T09:12:13.562 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60797.log 2024-09-14T09:12:13.562 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.55529.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61764.log 2024-09-14T09:12:13.562 INFO:teuthology.orchestra.run.smithi164.stderr: 87.4%/var/log/ceph/cluster2-client.mirror.60797.log: -- replaced with /var/log/ceph/cluster1-client.mirror.55529.log.gz 2024-09-14T09:12:13.562 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45368.log 2024-09-14T09:12:13.562 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55798.log 2024-09-14T09:12:13.562 INFO:teuthology.orchestra.run.smithi164.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.60797.log.gz 2024-09-14T09:12:13.562 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.61764.log: /var/log/ceph/cluster2-client.mirror.45368.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52439.log 2024-09-14T09:12:13.562 INFO:teuthology.orchestra.run.smithi164.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.61764.log.gz 2024-09-14T09:12:13.562 INFO:teuthology.orchestra.run.smithi164.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.45368.log.gz 2024-09-14T09:12:13.562 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.55798.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.55798.log.gz 2024-09-14T09:12:13.562 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.72933.log 2024-09-14T09:12:13.563 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.52439.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57989.log 2024-09-14T09:12:13.563 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.72933.log: 88.4%gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68079.log 2024-09-14T09:12:13.563 INFO:teuthology.orchestra.run.smithi164.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.52439.log.gz 2024-09-14T09:12:13.563 INFO:teuthology.orchestra.run.smithi164.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.72933.log.gz 2024-09-14T09:12:13.563 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44308.log 2024-09-14T09:12:13.563 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.57989.log: /var/log/ceph/cluster1-client.mirror.68079.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65405.log 2024-09-14T09:12:13.563 INFO:teuthology.orchestra.run.smithi164.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.57989.log.gz 2024-09-14T09:12:13.563 INFO:teuthology.orchestra.run.smithi164.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.68079.log.gz 2024-09-14T09:12:13.563 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.44308.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.35750.log 2024-09-14T09:12:13.563 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.65405.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61295.log 2024-09-14T09:12:13.563 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.admin.35750.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.35750.log.gz 2024-09-14T09:12:13.563 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51193.log 2024-09-14T09:12:13.563 INFO:teuthology.orchestra.run.smithi164.stderr: 90.1% -- replaced with /var/log/ceph/cluster2-client.mirror.65405.log.gz 2024-09-14T09:12:13.563 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.61295.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66414.log 2024-09-14T09:12:13.564 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.51193.log: 87.3% -- replaced with /var/log/ceph/cluster2-client.mirror.51193.log.gz 2024-09-14T09:12:13.564 INFO:teuthology.orchestra.run.smithi164.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.61295.log.gz 2024-09-14T09:12:13.564 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67134.log 2024-09-14T09:12:13.564 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.66414.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61937.log 2024-09-14T09:12:13.564 INFO:teuthology.orchestra.run.smithi164.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.66414.log.gz 2024-09-14T09:12:13.564 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.67134.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52577.log 2024-09-14T09:12:13.564 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.61937.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.67134.log.gz 2024-09-14T09:12:13.564 INFO:teuthology.orchestra.run.smithi164.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.61937.log.gz 2024-09-14T09:12:13.564 INFO:teuthology.orchestra.run.smithi164.stderr: 93.4% -- replaced with /var/log/ceph/cluster2-client.mirror.44308.log.gz 2024-09-14T09:12:13.564 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69419.log 2024-09-14T09:12:13.564 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.52577.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46289.log 2024-09-14T09:12:13.564 INFO:teuthology.orchestra.run.smithi164.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.52577.log.gz 2024-09-14T09:12:13.564 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.69419.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.40314.log 2024-09-14T09:12:13.564 INFO:teuthology.orchestra.run.smithi164.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.69419.log.gz 2024-09-14T09:12:13.565 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.73348.log 2024-09-14T09:12:13.565 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.46289.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54677.log 2024-09-14T09:12:13.565 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.0.40314.log: /var/log/ceph/cluster1-client.mirror.73348.log: 87.2% -- replaced with /var/log/ceph/cluster1-client.mirror.73348.log.gz 2024-09-14T09:12:13.565 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58262.log 2024-09-14T09:12:13.565 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.54677.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.54677.log.gz 2024-09-14T09:12:13.565 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40732.log 2024-09-14T09:12:13.565 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.58262.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.58262.log.gz 2024-09-14T09:12:13.565 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.73986.log 2024-09-14T09:12:13.565 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.40732.log: 93.2% -- replaced with /var/log/ceph/cluster1-client.mirror.0.40314.log.gz 2024-09-14T09:12:13.565 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57411.log 2024-09-14T09:12:13.565 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.admin.73986.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.73986.log.gz 2024-09-14T09:12:13.565 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57816.log 2024-09-14T09:12:13.565 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.57411.log: 92.2% -- replaced with /var/log/ceph/cluster1-client.mirror.57411.log.gz 2024-09-14T09:12:13.565 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47766.log 2024-09-14T09:12:13.566 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.57816.log: 93.4% -- replaced with /var/log/ceph/cluster2-client.mirror.40732.log.gz 2024-09-14T09:12:13.566 INFO:teuthology.orchestra.run.smithi164.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.57816.log.gz 2024-09-14T09:12:13.566 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65881.log 2024-09-14T09:12:13.566 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.47766.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72643.log 2024-09-14T09:12:13.566 INFO:teuthology.orchestra.run.smithi164.stderr: 79.8% -- replaced with /var/log/ceph/cluster2-client.mirror.47766.log.gz 2024-09-14T09:12:13.566 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.65881.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.74320.log 2024-09-14T09:12:13.566 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.72643.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.65881.log.gz 2024-09-14T09:12:13.566 INFO:teuthology.orchestra.run.smithi164.stderr: 86.9% -- replaced with /var/log/ceph/cluster2-client.mirror.72643.log.gz 2024-09-14T09:12:13.566 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63162.log 2024-09-14T09:12:13.566 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.admin.74320.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.74320.log.gzgzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61913.log 2024-09-14T09:12:13.566 INFO:teuthology.orchestra.run.smithi164.stderr: 2024-09-14T09:12:13.566 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.74362.log 2024-09-14T09:12:13.566 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.63162.log: /var/log/ceph/cluster1-client.mirror.61913.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.61913.log.gz 2024-09-14T09:12:13.566 INFO:teuthology.orchestra.run.smithi164.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.63162.log.gz 2024-09-14T09:12:13.567 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.41302.log 2024-09-14T09:12:13.567 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63844.log 2024-09-14T09:12:13.579 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.admin.74362.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.74362.log.gz 2024-09-14T09:12:13.580 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.3.41302.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73558.log 2024-09-14T09:12:13.580 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.63844.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.63844.log.gz 2024-09-14T09:12:13.580 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55843.log 2024-09-14T09:12:13.580 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.73558.log: 87.1% -- replaced with /var/log/ceph/cluster2-client.mirror.73558.log.gz 2024-09-14T09:12:13.580 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69624.log 2024-09-14T09:12:13.580 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.55843.log: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.47066.log.gz 2024-09-14T09:12:13.580 INFO:teuthology.orchestra.run.smithi164.stderr: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.55843.log.gz 2024-09-14T09:12:13.580 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64806.log 2024-09-14T09:12:13.580 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.69624.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.37382.log 2024-09-14T09:12:13.580 INFO:teuthology.orchestra.run.smithi164.stderr: 79.6% -- replaced with /var/log/ceph/cluster2-client.mirror.69624.log.gz 2024-09-14T09:12:13.580 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.64806.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65963.log 2024-09-14T09:12:13.580 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.admin.37382.log: 66.2% -- replaced with /var/log/ceph/cluster1-client.admin.37382.log.gz 2024-09-14T09:12:13.580 INFO:teuthology.orchestra.run.smithi164.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.64806.log.gz 2024-09-14T09:12:13.581 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57081.log 2024-09-14T09:12:13.581 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.65963.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49531.log 2024-09-14T09:12:13.581 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.57081.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.57081.log.gz 2024-09-14T09:12:13.581 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.62026.log 2024-09-14T09:12:13.581 INFO:teuthology.orchestra.run.smithi164.stderr: 90.1% -- replaced with /var/log/ceph/cluster2-client.mirror.65963.log.gz 2024-09-14T09:12:13.581 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.49531.log: 79.6% -- replaced with /var/log/ceph/cluster2-client.mirror.49531.log.gz 2024-09-14T09:12:13.581 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65013.log 2024-09-14T09:12:13.581 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70746.log 2024-09-14T09:12:13.581 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.62026.log: /var/log/ceph/cluster2-client.mirror.65013.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.65013.log.gz 2024-09-14T09:12:13.581 INFO:teuthology.orchestra.run.smithi164.stderr: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.62026.log.gz 2024-09-14T09:12:13.581 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42059.log 2024-09-14T09:12:13.581 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.70746.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43338.log 2024-09-14T09:12:13.581 INFO:teuthology.orchestra.run.smithi164.stderr: 87.2% -- replaced with /var/log/ceph/cluster1-client.mirror.70746.log.gz 2024-09-14T09:12:13.581 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.42059.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37501.log 2024-09-14T09:12:13.582 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.43338.log: 89.0% -- replaced with /var/log/ceph/cluster1-client.mirror.42059.log.gz 2024-09-14T09:12:13.582 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.71392.log 2024-09-14T09:12:13.582 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.37501.log: 89.6% 54.2% -- replaced with /var/log/ceph/cluster2-client.mirror.43338.log.gz -- replaced with /var/log/ceph/cluster1-client.mirror.37501.log.gz 2024-09-14T09:12:13.582 INFO:teuthology.orchestra.run.smithi164.stderr: 2024-09-14T09:12:13.582 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50703.log 2024-09-14T09:12:13.582 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5/var/log/ceph/cluster1-client.mirror.71392.log: --verbose -- /var/log/ceph/cluster1-client.mirror.56702.log 2024-09-14T09:12:13.582 INFO:teuthology.orchestra.run.smithi164.stderr: 87.2% -- replaced with /var/log/ceph/cluster1-client.mirror.71392.log.gz 2024-09-14T09:12:13.582 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.50703.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64488.log 2024-09-14T09:12:13.582 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.56702.log: 87.3% -- replaced with /var/log/ceph/cluster2-client.mirror.50703.log.gz 2024-09-14T09:12:13.582 INFO:teuthology.orchestra.run.smithi164.stderr: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.56702.log.gz 2024-09-14T09:12:13.582 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50036.log 2024-09-14T09:12:13.582 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50976.log 2024-09-14T09:12:13.582 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.64488.log: /var/log/ceph/cluster2-client.mirror.50036.log: 1.3% -- replaced with /var/log/ceph/cluster2-client.mirror.50036.log.gz 2024-09-14T09:12:13.582 INFO:teuthology.orchestra.run.smithi164.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.64488.log.gz 2024-09-14T09:12:13.583 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48982.log 2024-09-14T09:12:13.583 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50591.log 2024-09-14T09:12:13.597 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.50976.log: /var/log/ceph/cluster2-client.mirror.48982.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.50976.log.gz 2024-09-14T09:12:13.597 INFO:teuthology.orchestra.run.smithi164.stderr: 87.4%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39921.log 2024-09-14T09:12:13.597 INFO:teuthology.orchestra.run.smithi164.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.48982.log.gz 2024-09-14T09:12:13.597 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56995.log 2024-09-14T09:12:13.597 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.50591.log: /var/log/ceph/cluster2-client.mirror.39921.log: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.39921.log.gz 2024-09-14T09:12:13.597 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58684.log 2024-09-14T09:12:13.597 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.56995.log: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.50591.log.gz 2024-09-14T09:12:13.597 INFO:teuthology.orchestra.run.smithi164.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.56995.log.gz 2024-09-14T09:12:13.598 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69127.log 2024-09-14T09:12:13.598 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70040.log 2024-09-14T09:12:13.598 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.58684.log: /var/log/ceph/cluster2-client.mirror.69127.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.58684.log.gz 2024-09-14T09:12:13.598 INFO:teuthology.orchestra.run.smithi164.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.69127.log.gz 2024-09-14T09:12:13.598 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53782.log 2024-09-14T09:12:13.598 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.35914.log 2024-09-14T09:12:13.598 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.70040.log: /var/log/ceph/cluster2-client.mirror.53782.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.70040.log.gz 2024-09-14T09:12:13.598 INFO:teuthology.orchestra.run.smithi164.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.53782.log.gz 2024-09-14T09:12:13.598 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51135.log 2024-09-14T09:12:13.598 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose/var/log/ceph/cluster2-client.admin.35914.log: -- /var/log/ceph/cluster1-client.mirror.41655.log 2024-09-14T09:12:13.598 INFO:teuthology.orchestra.run.smithi164.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.35914.log.gz 2024-09-14T09:12:13.598 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72201.log 2024-09-14T09:12:13.598 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.51135.log: /var/log/ceph/cluster1-client.mirror.41655.log: 88.3% -- replaced with /var/log/ceph/cluster1-client.mirror.41655.log.gz 2024-09-14T09:12:13.599 INFO:teuthology.orchestra.run.smithi164.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.51135.log.gz 2024-09-14T09:12:13.599 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65364.log 2024-09-14T09:12:13.599 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.72201.log: 78.8%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43109.log 2024-09-14T09:12:13.599 INFO:teuthology.orchestra.run.smithi164.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.72201.log.gz 2024-09-14T09:12:13.599 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62576.log 2024-09-14T09:12:13.599 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.65364.log: /var/log/ceph/cluster2-client.mirror.43109.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.65364.log.gz 2024-09-14T09:12:13.599 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66819.log 2024-09-14T09:12:13.599 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.62576.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.62576.log.gz 2024-09-14T09:12:13.599 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53292.log 2024-09-14T09:12:13.599 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.66819.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.66819.log.gz 2024-09-14T09:12:13.599 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50952.log 2024-09-14T09:12:13.599 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.53292.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.53292.log.gz 2024-09-14T09:12:13.599 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41064.log 2024-09-14T09:12:13.599 INFO:teuthology.orchestra.run.smithi164.stderr: 93.4% -- replaced with /var/log/ceph/cluster2-client.mirror.43109.log.gz 2024-09-14T09:12:13.600 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.50952.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.48102.log 2024-09-14T09:12:13.600 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.41064.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.50952.log.gz 2024-09-14T09:12:13.600 INFO:teuthology.orchestra.run.smithi164.stderr: 88.7% -- replaced with /var/log/ceph/cluster1-client.mirror.41064.log.gz 2024-09-14T09:12:13.600 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61162.log 2024-09-14T09:12:13.600 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48684.log 2024-09-14T09:12:13.600 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.0.48102.log: /var/log/ceph/cluster1-client.mirror.61162.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.61162.log.gz 2024-09-14T09:12:13.600 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49833.log 2024-09-14T09:12:13.610 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.48684.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57515.log 2024-09-14T09:12:13.611 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.49833.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.49833.log.gz 2024-09-14T09:12:13.626 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.71182.log 2024-09-14T09:12:13.626 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.57515.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.57515.log.gz 2024-09-14T09:12:13.632 INFO:teuthology.orchestra.run.smithi164.stderr: 93.3% -- replaced with /var/log/ceph/cluster1-client.mirror.0.48102.log.gz 2024-09-14T09:12:13.634 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57105.log 2024-09-14T09:12:13.634 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.71182.log: 85.1% -- replaced with /var/log/ceph/cluster1-client.mirror.71182.log.gz 2024-09-14T09:12:13.634 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.36861.log 2024-09-14T09:12:13.635 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.57105.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.57105.log.gz 2024-09-14T09:12:13.635 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37746.log 2024-09-14T09:12:13.648 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.1.36861.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43698.log 2024-09-14T09:12:13.649 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.37746.log: 54.2% -- replaced with /var/log/ceph/cluster2-client.mirror.37746.log.gz 2024-09-14T09:12:13.662 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61000.log 2024-09-14T09:12:13.663 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.43698.log: 89.7% -- replaced with /var/log/ceph/cluster1-client.mirror.43698.log.gz 2024-09-14T09:12:13.664 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71542.log 2024-09-14T09:12:13.666 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.61000.log: 88.4% 92.4% -- replaced with /var/log/ceph/cluster2-client.mirror.61000.log.gz 2024-09-14T09:12:13.667 INFO:teuthology.orchestra.run.smithi164.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.1.36861.log.gz 2024-09-14T09:12:13.667 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55159.log 2024-09-14T09:12:13.667 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.71542.log: 87.0% -- replaced with /var/log/ceph/cluster2-client.mirror.71542.log.gz 2024-09-14T09:12:13.668 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39710.log 2024-09-14T09:12:13.669 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.55159.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.55159.log.gz 2024-09-14T09:12:13.670 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44523.log 2024-09-14T09:12:13.675 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.39710.log: 93.3% -- replaced with /var/log/ceph/cluster2-client.mirror.39710.log.gz 2024-09-14T09:12:13.676 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65674.log 2024-09-14T09:12:13.682 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.44523.log: 93.4% -- replaced with /var/log/ceph/cluster2-client.mirror.44523.log.gz 2024-09-14T09:12:13.682 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65591.log 2024-09-14T09:12:13.682 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.65674.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.65674.log.gz 2024-09-14T09:12:13.683 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57857.log 2024-09-14T09:12:13.685 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.65591.log: 90.2% -- replaced with /var/log/ceph/cluster2-client.mirror.65591.log.gz 2024-09-14T09:12:13.686 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58977.log 2024-09-14T09:12:13.687 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.57857.log: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.57857.log.gz 2024-09-14T09:12:13.688 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43722.log 2024-09-14T09:12:13.689 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.58977.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.58977.log.gz 2024-09-14T09:12:13.690 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52887.log 2024-09-14T09:12:13.690 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.43722.log: 89.4% -- replaced with /var/log/ceph/cluster1-client.mirror.43722.log.gz 2024-09-14T09:12:13.691 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.47825.log.gz 2024-09-14T09:12:13.691 INFO:teuthology.orchestra.run.smithi164.stderr: /var/log/ceph/cluster1-client.mirror.67944.log 2024-09-14T09:12:13.692 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2.audit.log 2024-09-14T09:12:13.692 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.52887.log: /var/log/ceph/cluster1-client.mirror.67944.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.67944.log.gz 2024-09-14T09:12:13.692 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48863.log 2024-09-14T09:12:13.693 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2.audit.log: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.52887.log.gz 2024-09-14T09:12:13.695 INFO:teuthology.orchestra.run.smithi164.stderr: 93.2% -- replaced with /var/log/ceph/cluster2.audit.log.gz 2024-09-14T09:12:13.696 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69357.log 2024-09-14T09:12:13.696 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42124.log 2024-09-14T09:12:13.697 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.48863.log: 79.9% -- replaced with /var/log/ceph/cluster2-client.mirror.48863.log.gz 2024-09-14T09:12:13.697 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.69357.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46870.log 2024-09-14T09:12:13.697 INFO:teuthology.orchestra.run.smithi164.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.69357.log.gz 2024-09-14T09:12:13.702 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.42124.log: 93.4% -- replaced with /var/log/ceph/cluster1-client.mirror.42124.log.gz 2024-09-14T09:12:13.703 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.42352.log 2024-09-14T09:12:13.703 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63224.log 2024-09-14T09:12:13.704 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.46870.log: /var/log/ceph/cluster1-client.admin.42352.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.46870.log.gz 2024-09-14T09:12:13.704 INFO:teuthology.orchestra.run.smithi164.stderr: 87.9% -- replaced with /var/log/ceph/cluster1-client.admin.42352.log.gz 2024-09-14T09:12:13.704 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67224.log 2024-09-14T09:12:13.705 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48945.log 2024-09-14T09:12:13.705 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.63224.log: /var/log/ceph/cluster1-client.mirror.67224.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.63224.log.gz 2024-09-14T09:12:13.705 INFO:teuthology.orchestra.run.smithi164.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.67224.log.gz 2024-09-14T09:12:13.705 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53932.log 2024-09-14T09:12:13.707 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.39958.log 2024-09-14T09:12:13.707 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.48945.log: 79.6% -- replaced with /var/log/ceph/cluster2-client.mirror.48945.log.gz 2024-09-14T09:12:13.707 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.53932.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47926.log 2024-09-14T09:12:13.707 INFO:teuthology.orchestra.run.smithi164.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.53932.log.gz 2024-09-14T09:12:13.715 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.0.39958.log: 92.7% -- replaced with /var/log/ceph/cluster1-client.mirror.3.41302.log.gz 2024-09-14T09:12:13.715 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46521.log 2024-09-14T09:12:13.718 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49670.log 2024-09-14T09:12:13.718 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.47926.log: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.0.39958.log.gz 2024-09-14T09:12:13.718 INFO:teuthology.orchestra.run.smithi164.stderr: 79.8% -- replaced with /var/log/ceph/cluster2-client.mirror.47926.log.gz 2024-09-14T09:12:13.718 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.46521.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63575.log 2024-09-14T09:12:13.718 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.49670.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.39641.log 2024-09-14T09:12:13.718 INFO:teuthology.orchestra.run.smithi164.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.46521.log.gz 2024-09-14T09:12:13.718 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.63575.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.39730.log 2024-09-14T09:12:13.718 INFO:teuthology.orchestra.run.smithi164.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.63575.log.gz 2024-09-14T09:12:13.720 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.39641.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44242.log 2024-09-14T09:12:13.720 INFO:teuthology.orchestra.run.smithi164.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.39641.log.gz 2024-09-14T09:12:13.720 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.39730.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60643.log 2024-09-14T09:12:13.723 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.44242.log: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.48252.log.gz 2024-09-14T09:12:13.723 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65261.log 2024-09-14T09:12:13.723 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.60643.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.60643.log.gz 2024-09-14T09:12:13.726 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61478.log 2024-09-14T09:12:13.726 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.65261.log: 93.2% -- replaced with /var/log/ceph/cluster1-client.mirror.39730.log.gz 2024-09-14T09:12:13.726 INFO:teuthology.orchestra.run.smithi164.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.65261.log.gz 2024-09-14T09:12:13.726 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45475.log 2024-09-14T09:12:13.726 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.61478.log: 93.4% -- replaced with /var/log/ceph/cluster1-client.mirror.44242.log.gzgzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55043.log 2024-09-14T09:12:13.727 INFO:teuthology.orchestra.run.smithi164.stderr: 2024-09-14T09:12:13.727 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.61538.log 2024-09-14T09:12:13.727 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.45475.log: /var/log/ceph/cluster2-client.mirror.55043.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.55043.log.gz 2024-09-14T09:12:13.727 INFO:teuthology.orchestra.run.smithi164.stderr: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.61478.log.gz 2024-09-14T09:12:13.727 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44179.log 2024-09-14T09:12:13.727 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.0.61538.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56369.log 2024-09-14T09:12:13.732 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.44179.log: 93.4% -- replaced with /var/log/ceph/cluster2-client.mirror.44179.log.gz 2024-09-14T09:12:13.732 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47177.log 2024-09-14T09:12:13.733 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.56369.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.56369.log.gz 2024-09-14T09:12:13.734 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65467.log 2024-09-14T09:12:13.737 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.47177.log: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.39817.log.gz 2024-09-14T09:12:13.738 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56043.log 2024-09-14T09:12:13.739 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.65467.log: 90.1% -- replaced with /var/log/ceph/cluster2-client.mirror.65467.log.gz 2024-09-14T09:12:13.740 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71349.log 2024-09-14T09:12:13.740 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.56043.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.56043.log.gz 2024-09-14T09:12:13.741 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43088.log 2024-09-14T09:12:13.742 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.71349.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.71349.log.gz 2024-09-14T09:12:13.742 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37661.log 2024-09-14T09:12:13.743 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.43088.log: 88.4% -- replaced with /var/log/ceph/cluster1-client.mirror.43088.log.gz 2024-09-14T09:12:13.745 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36837.log 2024-09-14T09:12:13.745 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.37661.log: 54.4% -- replaced with /var/log/ceph/cluster2-client.mirror.37661.log.gz 2024-09-14T09:12:13.745 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.59220.log 2024-09-14T09:12:13.748 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.36837.log: 94.0% -- replaced with /var/log/ceph/cluster2-client.mirror.0.61538.log.gz 2024-09-14T09:12:13.751 INFO:teuthology.orchestra.run.smithi164.stderr: 94.5% -- replaced with /var/log/ceph/cluster2-client.mirror.36837.log.gz 2024-09-14T09:12:13.752 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63057.log 2024-09-14T09:12:13.752 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.1.59220.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69793.log 2024-09-14T09:12:13.754 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.63057.log: 90.2% -- replaced with /var/log/ceph/cluster2-client.mirror.63057.log.gz 2024-09-14T09:12:13.754 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.72499.log 2024-09-14T09:12:13.756 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.69793.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.69793.log.gz 2024-09-14T09:12:13.756 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.36758.log 2024-09-14T09:12:13.756 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.72499.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.72499.log.gz 2024-09-14T09:12:13.758 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.40752.log 2024-09-14T09:12:13.758 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.admin.36758.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.36758.log.gz 2024-09-14T09:12:13.758 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43964.log 2024-09-14T09:12:13.763 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.40752.log: 93.4% -- replaced with /var/log/ceph/cluster1-client.mirror.40752.log.gz 2024-09-14T09:12:13.763 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54951.log 2024-09-14T09:12:13.769 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.43964.log: 93.4% -- replaced with /var/log/ceph/cluster2-client.mirror.43964.log.gz 2024-09-14T09:12:13.770 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71707.log 2024-09-14T09:12:13.772 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.54951.log: 92.4% -- replaced with /var/log/ceph/cluster2-client.mirror.54951.log.gz 2024-09-14T09:12:13.772 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.40663.log 2024-09-14T09:12:13.773 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.71707.log: 86.9% -- replaced with /var/log/ceph/cluster2-client.mirror.71707.log.gz 2024-09-14T09:12:13.774 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48962.log 2024-09-14T09:12:13.774 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.40663.log: 88.3% -- replaced with /var/log/ceph/cluster1-client.mirror.40663.log.gz 2024-09-14T09:12:13.775 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50546.log 2024-09-14T09:12:13.776 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.48962.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.48962.log.gz 2024-09-14T09:12:13.776 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41719.log 2024-09-14T09:12:13.777 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.50546.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.50546.log.gz 2024-09-14T09:12:13.777 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41761.log 2024-09-14T09:12:13.783 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.41719.log: 93.4% -- replaced with /var/log/ceph/cluster2-client.mirror.41719.log.gz 2024-09-14T09:12:13.784 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43129.log 2024-09-14T09:12:13.784 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.41761.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.41761.log.gz 2024-09-14T09:12:13.785 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66324.log 2024-09-14T09:12:13.790 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.43129.log: 93.3% -- replaced with /var/log/ceph/cluster1-client.mirror.43129.log.gz 2024-09-14T09:12:13.792 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.36506.log 2024-09-14T09:12:13.792 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.66324.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.66324.log.gz 2024-09-14T09:12:13.794 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64785.log 2024-09-14T09:12:13.794 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.admin.36506.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.36506.log.gz 2024-09-14T09:12:13.794 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65075.log 2024-09-14T09:12:13.796 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.64785.log: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.64785.log.gz 2024-09-14T09:12:13.796 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61500.log 2024-09-14T09:12:13.797 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.65075.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.65075.log.gz 2024-09-14T09:12:13.797 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72368.log 2024-09-14T09:12:13.798 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.61500.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.61500.log.gz 2024-09-14T09:12:13.798 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63451.log 2024-09-14T09:12:13.799 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.72368.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.72368.log.gz 2024-09-14T09:12:13.800 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64641.log 2024-09-14T09:12:13.801 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.63451.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.63451.log.gz 2024-09-14T09:12:13.801 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68669.log 2024-09-14T09:12:13.802 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.64641.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.64641.log.gz 2024-09-14T09:12:13.803 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49293.log 2024-09-14T09:12:13.804 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.68669.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.68669.log.gz 2024-09-14T09:12:13.804 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54539.log 2024-09-14T09:12:13.805 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.49293.log: 79.7% -- replaced with /var/log/ceph/cluster2-client.mirror.49293.log.gz 2024-09-14T09:12:13.805 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52304.log 2024-09-14T09:12:13.806 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.54539.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.54539.log.gz 2024-09-14T09:12:13.806 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43016.log 2024-09-14T09:12:13.807 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.52304.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.52304.log.gz 2024-09-14T09:12:13.808 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44093.log 2024-09-14T09:12:13.808 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.43016.log: 89.1% -- replaced with /var/log/ceph/cluster1-client.mirror.43016.log.gz 2024-09-14T09:12:13.809 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60712.log 2024-09-14T09:12:13.815 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.44093.log: 93.4% -- replaced with /var/log/ceph/cluster2-client.mirror.44093.log.gz 2024-09-14T09:12:13.815 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65137.log 2024-09-14T09:12:13.816 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.60712.log: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.48684.log.gz 2024-09-14T09:12:13.817 INFO:teuthology.orchestra.run.smithi164.stderr: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.60712.log.gz 2024-09-14T09:12:13.817 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37416.log 2024-09-14T09:12:13.820 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose/var/log/ceph/cluster2-client.mirror.65137.log: -- /var/log/ceph/cluster1-client.mirror.68192.log 2024-09-14T09:12:13.820 INFO:teuthology.orchestra.run.smithi164.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.65137.log.gz 2024-09-14T09:12:13.820 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.37416.log: 2.3% -- replaced with /var/log/ceph/cluster1-client.mirror.37416.log.gz 2024-09-14T09:12:13.820 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58753.log 2024-09-14T09:12:13.820 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.68192.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41220.log 2024-09-14T09:12:13.820 INFO:teuthology.orchestra.run.smithi164.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.68192.log.gz 2024-09-14T09:12:13.821 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.58753.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51310.log 2024-09-14T09:12:13.821 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.41220.log: 92.3% -- replaced with /var/log/ceph/cluster2-client.mirror.58753.log.gz 2024-09-14T09:12:13.822 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56953.log 2024-09-14T09:12:13.824 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.51310.log: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.51310.log.gz 2024-09-14T09:12:13.824 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37763.log 2024-09-14T09:12:13.827 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.56953.log: 93.4% -- replaced with /var/log/ceph/cluster1-client.mirror.41220.log.gz 2024-09-14T09:12:13.827 INFO:teuthology.orchestra.run.smithi164.stderr: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.56953.log.gz 2024-09-14T09:12:13.827 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37802.log 2024-09-14T09:12:13.827 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53071.log 2024-09-14T09:12:13.827 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.37763.log: 54.2% -- replaced with /var/log/ceph/cluster2-client.mirror.37763.log.gz 2024-09-14T09:12:13.829 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.37802.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71625.log 2024-09-14T09:12:13.829 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.53071.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.37802.log.gz 2024-09-14T09:12:13.829 INFO:teuthology.orchestra.run.smithi164.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.53071.log.gz 2024-09-14T09:12:13.829 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69227.log 2024-09-14T09:12:13.829 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.71625.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.40873.log 2024-09-14T09:12:13.829 INFO:teuthology.orchestra.run.smithi164.stderr: 87.0% -- replaced with /var/log/ceph/cluster2-client.mirror.71625.log.gz 2024-09-14T09:12:13.830 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.69227.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50775.log 2024-09-14T09:12:13.831 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.admin.40873.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.40873.log.gz 2024-09-14T09:12:13.832 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44457.log 2024-09-14T09:12:13.832 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.50775.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.50775.log.gz 2024-09-14T09:12:13.833 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48721.log 2024-09-14T09:12:13.839 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.44457.log: 93.4% -- replaced with /var/log/ceph/cluster1-client.mirror.44457.log.gz 2024-09-14T09:12:13.839 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37678.log 2024-09-14T09:12:13.841 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.48721.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.48721.log.gz 2024-09-14T09:12:13.841 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67292.log 2024-09-14T09:12:13.841 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.37678.log: 3.5% -- replaced with /var/log/ceph/cluster2-client.mirror.37678.log.gz 2024-09-14T09:12:13.843 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47544.log 2024-09-14T09:12:13.843 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.67292.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.67292.log.gz 2024-09-14T09:12:13.843 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56630.log 2024-09-14T09:12:13.844 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.47544.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.47544.log.gz 2024-09-14T09:12:13.845 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47883.log 2024-09-14T09:12:13.845 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.56630.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.56630.log.gz 2024-09-14T09:12:13.846 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61648.log 2024-09-14T09:12:13.847 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.47883.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.47883.log.gz 2024-09-14T09:12:13.847 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47484.log 2024-09-14T09:12:13.848 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.61648.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.61648.log.gz 2024-09-14T09:12:13.849 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46247.log 2024-09-14T09:12:13.850 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.47484.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.47484.log.gz 2024-09-14T09:12:13.850 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.36632.log 2024-09-14T09:12:13.852 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.46247.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.46247.log.gz 2024-09-14T09:12:13.852 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71646.log 2024-09-14T09:12:13.853 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.admin.36632.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.36632.log.gz 2024-09-14T09:12:13.853 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55774.log 2024-09-14T09:12:13.854 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.71646.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.71646.log.gz 2024-09-14T09:12:13.854 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51470.log 2024-09-14T09:12:13.855 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.55774.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.55774.log.gz 2024-09-14T09:12:13.856 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56973.log 2024-09-14T09:12:13.857 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.51470.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.51470.log.gz 2024-09-14T09:12:13.857 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.62648.log 2024-09-14T09:12:13.859 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.56973.log: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.56973.log.gz 2024-09-14T09:12:13.859 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41963.log 2024-09-14T09:12:13.861 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.62648.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.62648.log.gz 2024-09-14T09:12:13.861 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54906.log 2024-09-14T09:12:13.862 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.41963.log: 88.9% -- replaced with /var/log/ceph/cluster1-client.mirror.41963.log.gz 2024-09-14T09:12:13.862 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49548.log 2024-09-14T09:12:13.863 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.54906.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.54906.log.gz 2024-09-14T09:12:13.863 INFO:teuthology.orchestra.run.smithi164.stderr: 93.3% -- replaced with /var/log/ceph/cluster1-client.mirror.1.59220.log.gzgzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.40189.log 2024-09-14T09:12:13.863 INFO:teuthology.orchestra.run.smithi164.stderr: 2024-09-14T09:12:13.866 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.49548.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64827.log 2024-09-14T09:12:13.866 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.40189.log: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.40189.log.gz 2024-09-14T09:12:13.866 INFO:teuthology.orchestra.run.smithi164.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.49548.log.gz 2024-09-14T09:12:13.867 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48625.log 2024-09-14T09:12:13.867 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.64827.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57370.log 2024-09-14T09:12:13.867 INFO:teuthology.orchestra.run.smithi164.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.64827.log.gz 2024-09-14T09:12:13.867 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.48625.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56548.log 2024-09-14T09:12:13.867 INFO:teuthology.orchestra.run.smithi164.stderr: 79.7% -- replaced with /var/log/ceph/cluster2-client.mirror.48625.log.gz 2024-09-14T09:12:13.867 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.57370.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.57370.log.gz 2024-09-14T09:12:13.875 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58190.log 2024-09-14T09:12:13.875 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.56548.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54380.log 2024-09-14T09:12:13.875 INFO:teuthology.orchestra.run.smithi164.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.56548.log.gz/var/log/ceph/cluster2-client.mirror.58190.log: 2024-09-14T09:12:13.875 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50233.log 2024-09-14T09:12:13.875 INFO:teuthology.orchestra.run.smithi164.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.58190.log.gz 2024-09-14T09:12:13.875 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.54380.log: gzip -5 --verbose 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.54380.log.gz 2024-09-14T09:12:13.876 INFO:teuthology.orchestra.run.smithi164.stderr: -- /var/log/ceph/cluster2-client.mirror.65715.log 2024-09-14T09:12:13.876 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44872.log 2024-09-14T09:12:13.876 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.50233.log: /var/log/ceph/cluster2-client.mirror.65715.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.50233.log.gz 2024-09-14T09:12:13.876 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68971.log 2024-09-14T09:12:13.876 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.44872.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.44872.log.gz 2024-09-14T09:12:13.876 INFO:teuthology.orchestra.run.smithi164.stderr: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.65715.log.gz 2024-09-14T09:12:13.876 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46504.log 2024-09-14T09:12:13.876 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.68971.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60755.log 2024-09-14T09:12:13.876 INFO:teuthology.orchestra.run.smithi164.stderr: 78.3% -- replaced with /var/log/ceph/cluster1-client.mirror.68971.log.gz 2024-09-14T09:12:13.876 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.46504.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66774.log 2024-09-14T09:12:13.876 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.60755.log: 79.7% -- replaced with /var/log/ceph/cluster2-client.mirror.46504.log.gz 2024-09-14T09:12:13.876 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71371.log 2024-09-14T09:12:13.877 INFO:teuthology.orchestra.run.smithi164.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.60755.log.gz 2024-09-14T09:12:13.877 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.66774.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39397.log 2024-09-14T09:12:13.877 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.71371.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.66774.log.gz 2024-09-14T09:12:13.877 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64558.log 2024-09-14T09:12:13.877 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.39397.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.39397.log.gz 2024-09-14T09:12:13.878 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53956.log 2024-09-14T09:12:13.878 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.64558.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.64558.log.gz 2024-09-14T09:12:13.878 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73768.log 2024-09-14T09:12:13.878 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.53956.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.53956.log.gz 2024-09-14T09:12:13.878 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.36590.log 2024-09-14T09:12:13.879 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.73768.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.73768.log.gz 2024-09-14T09:12:13.879 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49935.log 2024-09-14T09:12:13.879 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.admin.36590.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.36590.log.gz 2024-09-14T09:12:13.880 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.73596.log 2024-09-14T09:12:13.889 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.49935.log: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.45475.log.gz 2024-09-14T09:12:13.890 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72912.log 2024-09-14T09:12:13.892 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.3.73596.log: 91.3% -- replaced with /var/log/ceph/cluster1-client.mirror.3.73596.log.gz 2024-09-14T09:12:13.892 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37002.log 2024-09-14T09:12:13.892 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.72912.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.72912.log.gz 2024-09-14T09:12:13.893 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.74152.log 2024-09-14T09:12:13.894 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.37002.log: 91.4% -- replaced with /var/log/ceph/cluster1-client.mirror.37002.log.gz 2024-09-14T09:12:13.894 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67022.log 2024-09-14T09:12:13.896 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.admin.74152.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.74152.log.gz 2024-09-14T09:12:13.896 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73018.log 2024-09-14T09:12:13.897 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.67022.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.67022.log.gz 2024-09-14T09:12:13.897 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47623.log 2024-09-14T09:12:13.898 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.73018.log: 87.0% -- replaced with /var/log/ceph/cluster2-client.mirror.73018.log.gz 2024-09-14T09:12:13.898 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.36041.log 2024-09-14T09:12:13.899 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.47623.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.47623.log.gz 2024-09-14T09:12:13.900 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67607.log 2024-09-14T09:12:13.900 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.admin.36041.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.36041.log.gz 2024-09-14T09:12:13.902 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73941.log 2024-09-14T09:12:13.902 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.67607.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.67607.log.gz 2024-09-14T09:12:13.903 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67494.log 2024-09-14T09:12:13.903 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.73941.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.mirror.73941.log.gz 2024-09-14T09:12:13.904 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64405.log 2024-09-14T09:12:13.905 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.67494.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.67494.log.gz 2024-09-14T09:12:13.905 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.71937.log 2024-09-14T09:12:13.906 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.64405.log: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.64405.log.gz 2024-09-14T09:12:13.907 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70976.log 2024-09-14T09:12:13.909 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.71937.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.71937.log.gz 2024-09-14T09:12:13.909 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55421.log 2024-09-14T09:12:13.909 INFO:teuthology.orchestra.run.smithi164.stderr: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.49670.log.gz 2024-09-14T09:12:13.909 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.70976.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48466.log 2024-09-14T09:12:13.909 INFO:teuthology.orchestra.run.smithi164.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.70976.log.gz 2024-09-14T09:12:13.910 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.55421.log: 92.2% -- replaced with /var/log/ceph/cluster1-client.mirror.55421.log.gz 2024-09-14T09:12:13.914 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72623.log 2024-09-14T09:12:13.914 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.48466.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58708.log 2024-09-14T09:12:13.914 INFO:teuthology.orchestra.run.smithi164.stderr: 79.5% -- replaced with /var/log/ceph/cluster2-client.mirror.48466.log.gz 2024-09-14T09:12:13.914 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.72623.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51581.log 2024-09-14T09:12:13.914 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.58708.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.72623.log.gz 2024-09-14T09:12:13.914 INFO:teuthology.orchestra.run.smithi164.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.58708.log.gz 2024-09-14T09:12:13.914 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60487.log 2024-09-14T09:12:13.914 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45453.log 2024-09-14T09:12:13.915 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.60487.log: /var/log/ceph/cluster2-client.mirror.51581.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.51581.log.gz 2024-09-14T09:12:13.915 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64765.log 2024-09-14T09:12:13.915 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.45453.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.45453.log.gz 2024-09-14T09:12:13.916 INFO:teuthology.orchestra.run.smithi164.stderr: 92.2% -- replaced with /var/log/ceph/cluster1-client.mirror.60487.log.gz 2024-09-14T09:12:13.916 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58443.log 2024-09-14T09:12:13.917 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.36987.log 2024-09-14T09:12:13.917 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.64765.log: /var/log/ceph/cluster1-client.mirror.58443.log: 86.8% -- replaced with /var/log/ceph/cluster2-client.mirror.64765.log.gz 2024-09-14T09:12:13.917 INFO:teuthology.orchestra.run.smithi164.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.58443.log.gzgzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46992.log 2024-09-14T09:12:13.917 INFO:teuthology.orchestra.run.smithi164.stderr: 2024-09-14T09:12:13.919 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41177.log 2024-09-14T09:12:13.919 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.0.36987.log: /var/log/ceph/cluster2-client.mirror.46992.log: 71.0% -- replaced with /var/log/ceph/cluster2-client.mirror.0.36987.log.gz 2024-09-14T09:12:13.919 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41676.log 2024-09-14T09:12:13.925 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.41177.log: 93.4% -- replaced with /var/log/ceph/cluster1-client.mirror.41177.log.gz 2024-09-14T09:12:13.925 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.37570.log 2024-09-14T09:12:13.930 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.41676.log: 93.7% 93.4% -- replaced with /var/log/ceph/cluster2-client.mirror.41676.log.gz 2024-09-14T09:12:13.930 INFO:teuthology.orchestra.run.smithi164.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.46289.log.gz 2024-09-14T09:12:13.930 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.49707.log 2024-09-14T09:12:13.932 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55750.log 2024-09-14T09:12:13.932 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.admin.37570.log: 80.1% -- replaced with /var/log/ceph/cluster2-client.admin.37570.log.gz 2024-09-14T09:12:13.951 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.0.49707.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59040.log 2024-09-14T09:12:13.952 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.55750.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.55750.log.gz 2024-09-14T09:12:13.964 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65653.log 2024-09-14T09:12:13.965 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.59040.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.59040.log.gz 2024-09-14T09:12:13.969 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50611.log 2024-09-14T09:12:13.970 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.65653.log: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.0.49707.log.gz 2024-09-14T09:12:13.971 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43543.log 2024-09-14T09:12:13.971 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.50611.log: 90.1% -- replaced with /var/log/ceph/cluster2-client.mirror.65653.log.gz 2024-09-14T09:12:13.972 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63016.log 2024-09-14T09:12:13.973 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.43543.log: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.50611.log.gz 2024-09-14T09:12:13.973 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54476.log 2024-09-14T09:12:13.974 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.63016.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.63016.log.gz 2024-09-14T09:12:13.975 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51402.log 2024-09-14T09:12:13.976 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.54476.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.54476.log.gz 2024-09-14T09:12:13.976 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45073.log 2024-09-14T09:12:13.978 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.51402.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.51402.log.gz 2024-09-14T09:12:13.978 INFO:teuthology.orchestra.run.smithi164.stderr: 93.4% -- replaced with /var/log/ceph/cluster1-client.mirror.43543.log.gz 2024-09-14T09:12:13.978 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51072.log 2024-09-14T09:12:13.978 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose/var/log/ceph/cluster2-client.mirror.45073.log: -- /var/log/ceph/cluster2-client.mirror.70458.log 2024-09-14T09:12:13.979 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.51072.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.51072.log.gz 2024-09-14T09:12:13.979 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52142.log 2024-09-14T09:12:13.979 INFO:teuthology.orchestra.run.smithi164.stderr: 90.3% -- replaced with /var/log/ceph/cluster2-client.mirror.45073.log.gz 2024-09-14T09:12:13.980 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48900.log 2024-09-14T09:12:13.983 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.70458.log: /var/log/ceph/cluster1-client.mirror.52142.log: 86.9% -- replaced with /var/log/ceph/cluster2-client.mirror.70458.log.gz 2024-09-14T09:12:13.984 INFO:teuthology.orchestra.run.smithi164.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.52142.log.gz 2024-09-14T09:12:13.984 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67967.log 2024-09-14T09:12:13.984 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.48900.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37621.log 2024-09-14T09:12:13.984 INFO:teuthology.orchestra.run.smithi164.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.48900.log.gz 2024-09-14T09:12:13.984 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.67967.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60889.log 2024-09-14T09:12:13.984 INFO:teuthology.orchestra.run.smithi164.stderr: 86.7%/var/log/ceph/cluster1-client.mirror.37621.log: -- replaced with /var/log/ceph/cluster1-client.mirror.67967.log.gz 2024-09-14T09:12:13.984 INFO:teuthology.orchestra.run.smithi164.stderr: 73.9% -- replaced with /var/log/ceph/cluster1-client.mirror.37621.log.gz 2024-09-14T09:12:13.984 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47905.log 2024-09-14T09:12:13.984 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose --/var/log/ceph/cluster1-client.mirror.60889.log: /var/log/ceph/cluster2-client.mirror.58578.log 2024-09-14T09:12:13.984 INFO:teuthology.orchestra.run.smithi164.stderr: 87.2% -- replaced with /var/log/ceph/cluster1-client.mirror.60889.log.gz 2024-09-14T09:12:14.003 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.47905.log: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.47177.log.gz 2024-09-14T09:12:14.004 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.38645.log 2024-09-14T09:12:14.004 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.58578.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46789.log 2024-09-14T09:12:14.006 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.admin.38645.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.58578.log.gz 2024-09-14T09:12:14.007 INFO:teuthology.orchestra.run.smithi164.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.38645.log.gz 2024-09-14T09:12:14.007 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50099.log 2024-09-14T09:12:14.007 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.46789.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41511.log 2024-09-14T09:12:14.007 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.50099.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.46789.log.gz 1.3% -- replaced with /var/log/ceph/cluster1-client.mirror.50099.log.gz 2024-09-14T09:12:14.007 INFO:teuthology.orchestra.run.smithi164.stderr: 2024-09-14T09:12:14.007 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49352.log 2024-09-14T09:12:14.014 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose/var/log/ceph/cluster2-client.mirror.41511.log: -- /var/log/ceph/cluster1-client.mirror.68327.log 2024-09-14T09:12:14.014 INFO:teuthology.orchestra.run.smithi164.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.41511.log.gz 2024-09-14T09:12:14.014 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.49352.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37467.log 2024-09-14T09:12:14.014 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.68327.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.68327.log.gz 2024-09-14T09:12:14.015 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54653.log 2024-09-14T09:12:14.015 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.37467.log: 57.1% -- replaced with /var/log/ceph/cluster1-client.mirror.37467.log.gz 2024-09-14T09:12:14.015 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.36674.log 2024-09-14T09:12:14.015 INFO:teuthology.orchestra.run.smithi164.stderr: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.49935.log.gz 2024-09-14T09:12:14.015 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.54653.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57471.log 2024-09-14T09:12:14.015 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.admin.36674.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.54653.log.gz 2024-09-14T09:12:14.015 INFO:teuthology.orchestra.run.smithi164.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.36674.log.gz 2024-09-14T09:12:14.015 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69439.log 2024-09-14T09:12:14.015 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose/var/log/ceph/cluster2-client.mirror.57471.log: -- /var/log/ceph/cluster2-client.mirror.45505.log 2024-09-14T09:12:14.015 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.69439.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.57471.log.gz 2024-09-14T09:12:14.015 INFO:teuthology.orchestra.run.smithi164.stderr: 87.0% -- replaced with /var/log/ceph/cluster2-client.mirror.69439.log.gz 2024-09-14T09:12:14.015 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65571.log 2024-09-14T09:12:14.015 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.45505.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73455.log 2024-09-14T09:12:14.015 INFO:teuthology.orchestra.run.smithi164.stderr: 79.8% -- replaced with /var/log/ceph/cluster2-client.mirror.45505.log.gz 2024-09-14T09:12:14.016 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.65571.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43406.log 2024-09-14T09:12:14.016 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.73455.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.65571.log.gz 2024-09-14T09:12:14.020 INFO:teuthology.orchestra.run.smithi164.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.73455.log.gz 2024-09-14T09:12:14.020 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44712.log 2024-09-14T09:12:14.020 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose --/var/log/ceph/cluster1-client.mirror.43406.log: /var/log/ceph/cluster2-client.mirror.65426.log 2024-09-14T09:12:14.020 INFO:teuthology.orchestra.run.smithi164.stderr: 89.5% -- replaced with /var/log/ceph/cluster1-client.mirror.43406.log.gz/var/log/ceph/cluster2-client.mirror.44712.log: 2024-09-14T09:12:14.020 INFO:teuthology.orchestra.run.smithi164.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.44712.log.gz 2024-09-14T09:12:14.020 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66639.log 2024-09-14T09:12:14.020 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.65426.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49213.log 2024-09-14T09:12:14.020 INFO:teuthology.orchestra.run.smithi164.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.65426.log.gz/var/log/ceph/cluster1-client.mirror.66639.log: 2024-09-14T09:12:14.021 INFO:teuthology.orchestra.run.smithi164.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.66639.log.gz 2024-09-14T09:12:14.021 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63968.log 2024-09-14T09:12:14.021 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.49213.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48763.log 2024-09-14T09:12:14.021 INFO:teuthology.orchestra.run.smithi164.stderr: 79.8% -- replaced with /var/log/ceph/cluster2-client.mirror.49213.log.gz 2024-09-14T09:12:14.021 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.63968.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60061.log 2024-09-14T09:12:14.021 INFO:teuthology.orchestra.run.smithi164.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.63968.log.gz/var/log/ceph/cluster2-client.mirror.48763.log: 2024-09-14T09:12:14.033 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.73841.log 2024-09-14T09:12:14.034 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.60061.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.60061.log.gz 2024-09-14T09:12:14.049 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52601.log 2024-09-14T09:12:14.049 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.73841.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.mirror.73841.log.gz 2024-09-14T09:12:14.059 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.46600.log 2024-09-14T09:12:14.060 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.52601.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.52601.log.gz 2024-09-14T09:12:14.064 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69603.log 2024-09-14T09:12:14.066 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.46600.log: 94.7% -- replaced with /var/log/ceph/cluster1-client.mirror.46600.log.gz 2024-09-14T09:12:14.066 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.32542.log 2024-09-14T09:12:14.069 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.69603.log: 90.3% -- replaced with /var/log/ceph/cluster2-client.mirror.69603.log.gz 2024-09-14T09:12:14.070 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43878.log 2024-09-14T09:12:14.070 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.admin.32542.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.32542.log.gz 2024-09-14T09:12:14.070 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59960.log 2024-09-14T09:12:14.075 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.43878.log: 93.3% -- replaced with /var/log/ceph/cluster2-client.mirror.43878.log.gz 2024-09-14T09:12:14.076 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51788.log 2024-09-14T09:12:14.077 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.59960.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.59960.log.gz 2024-09-14T09:12:14.077 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43040.log 2024-09-14T09:12:14.080 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.51788.log: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.51788.log.gz 2024-09-14T09:12:14.080 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49489.log 2024-09-14T09:12:14.081 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.43040.log: 89.2% -- replaced with /var/log/ceph/cluster1-client.mirror.43040.log.gz 2024-09-14T09:12:14.081 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49992.log 2024-09-14T09:12:14.082 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.49489.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.49489.log.gz 2024-09-14T09:12:14.082 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73497.log 2024-09-14T09:12:14.086 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.49992.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.49992.log.gz 2024-09-14T09:12:14.086 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49913.log 2024-09-14T09:12:14.086 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.73497.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.73497.log.gz 2024-09-14T09:12:14.086 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44543.log 2024-09-14T09:12:14.087 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.49913.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.49913.log.gz 2024-09-14T09:12:14.087 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64217.log 2024-09-14T09:12:14.092 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.44543.log: 93.4% -- replaced with /var/log/ceph/cluster1-client.mirror.44543.log.gz 2024-09-14T09:12:14.093 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61538.log 2024-09-14T09:12:14.094 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.64217.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.64217.log.gz 2024-09-14T09:12:14.094 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73059.log 2024-09-14T09:12:14.095 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.61538.log: 94.4% -- replaced with /var/log/ceph/cluster1-client.mirror.61538.log.gz 2024-09-14T09:12:14.096 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51559.log 2024-09-14T09:12:14.097 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.73059.log: 87.0% -- replaced with /var/log/ceph/cluster2-client.mirror.73059.log.gz 2024-09-14T09:12:14.097 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63782.log 2024-09-14T09:12:14.099 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.51559.log: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.51559.log.gz 2024-09-14T09:12:14.100 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40447.log 2024-09-14T09:12:14.101 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.63782.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.63782.log.gz 2024-09-14T09:12:14.101 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.32755.log 2024-09-14T09:12:14.115 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.40447.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69398.log 2024-09-14T09:12:14.115 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.admin.32755.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.32755.log.gz 2024-09-14T09:12:14.128 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50329.log 2024-09-14T09:12:14.138 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.69398.log: 87.0% -- replaced with /var/log/ceph/cluster2-client.mirror.69398.log.gz 2024-09-14T09:12:14.139 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48922.log 2024-09-14T09:12:14.140 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.50329.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.50329.log.gz 2024-09-14T09:12:14.148 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60153.log 2024-09-14T09:12:14.158 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.48922.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50727.log 2024-09-14T09:12:14.158 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.60153.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.60153.log.gz 2024-09-14T09:12:14.166 INFO:teuthology.orchestra.run.smithi164.stderr: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.69227.log.gz 2024-09-14T09:12:14.166 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48023.log 2024-09-14T09:12:14.168 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.50727.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.50727.log.gz 2024-09-14T09:12:14.168 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47703.log 2024-09-14T09:12:14.170 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.48023.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.48023.log.gz 2024-09-14T09:12:14.170 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53832.log 2024-09-14T09:12:14.171 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.47703.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.47703.log.gz 2024-09-14T09:12:14.171 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53316.log 2024-09-14T09:12:14.172 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.53832.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.53832.log.gz 2024-09-14T09:12:14.173 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72331.log 2024-09-14T09:12:14.175 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.53316.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.53316.log.gz 2024-09-14T09:12:14.175 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-osd.0.log 2024-09-14T09:12:14.175 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.72331.log: 79.5% -- replaced with /var/log/ceph/cluster2-client.mirror.72331.log.gz 2024-09-14T09:12:14.175 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57837.log 2024-09-14T09:12:14.189 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-osd.0.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72582.log 2024-09-14T09:12:14.190 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.57837.log: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.57837.log.gz 2024-09-14T09:12:14.194 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.74194.log 2024-09-14T09:12:14.195 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.72582.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.72582.log.gz 2024-09-14T09:12:14.209 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60374.log 2024-09-14T09:12:14.209 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.admin.74194.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.74194.log.gz 2024-09-14T09:12:14.210 INFO:teuthology.orchestra.run.smithi164.stderr: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.48763.log.gz 2024-09-14T09:12:14.210 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54722.log 2024-09-14T09:12:14.212 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.60374.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.60374.log.gz 2024-09-14T09:12:14.212 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54971.log 2024-09-14T09:12:14.214 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.54722.log: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.54722.log.gz 2024-09-14T09:12:14.214 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62094.log 2024-09-14T09:12:14.217 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.54971.log: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.54971.log.gz 2024-09-14T09:12:14.217 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54133.log 2024-09-14T09:12:14.217 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.62094.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.62094.log.gz 2024-09-14T09:12:14.218 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72040.log 2024-09-14T09:12:14.221 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.54133.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.54133.log.gz 2024-09-14T09:12:14.221 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61827.log 2024-09-14T09:12:14.221 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.72040.log: 86.9% -- replaced with /var/log/ceph/cluster2-client.mirror.72040.log.gz 2024-09-14T09:12:14.221 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56236.log 2024-09-14T09:12:14.222 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.61827.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.61827.log.gz 2024-09-14T09:12:14.222 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71312.log 2024-09-14T09:12:14.223 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.56236.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.56236.log.gz 2024-09-14T09:12:14.225 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52215.log 2024-09-14T09:12:14.225 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.71312.log: 79.8% -- replaced with /var/log/ceph/cluster2-client.mirror.71312.log.gz 2024-09-14T09:12:14.225 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.35632.log 2024-09-14T09:12:14.227 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.52215.log: 95.3% -- replaced with /var/log/ceph/cluster1-client.mirror.52215.log.gz 2024-09-14T09:12:14.229 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73262.log 2024-09-14T09:12:14.229 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.admin.35632.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.35632.log.gz 2024-09-14T09:12:14.229 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65240.log 2024-09-14T09:12:14.230 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.73262.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.73262.log.gz 2024-09-14T09:12:14.230 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51651.log 2024-09-14T09:12:14.231 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.65240.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.65240.log.gz 2024-09-14T09:12:14.232 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60001.log 2024-09-14T09:12:14.233 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.51651.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.51651.log.gz 2024-09-14T09:12:14.233 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41242.log 2024-09-14T09:12:14.235 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.60001.log: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.60001.log.gz 2024-09-14T09:12:14.235 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48423.log 2024-09-14T09:12:14.237 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.41242.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.41242.log.gz 2024-09-14T09:12:14.237 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54497.log 2024-09-14T09:12:14.238 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.48423.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.48423.log.gz 2024-09-14T09:12:14.238 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66572.log 2024-09-14T09:12:14.240 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.54497.log: 92.4% -- replaced with /var/log/ceph/cluster2-client.mirror.54497.log.gz 2024-09-14T09:12:14.241 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45675.log 2024-09-14T09:12:14.241 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.66572.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.66572.log.gz 2024-09-14T09:12:14.243 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59354.log 2024-09-14T09:12:14.243 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.45675.log: 79.5% -- replaced with /var/log/ceph/cluster2-client.mirror.45675.log.gz 2024-09-14T09:12:14.243 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58037.log 2024-09-14T09:12:14.246 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.59354.log: 92.4% -- replaced with /var/log/ceph/cluster2-client.mirror.59354.log.gz 2024-09-14T09:12:14.246 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56281.log 2024-09-14T09:12:14.247 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.58037.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.58037.log.gz 2024-09-14T09:12:14.247 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70680.log 2024-09-14T09:12:14.249 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.56281.log: 93.7% 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.56281.log.gz 2024-09-14T09:12:14.250 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52053.log 2024-09-14T09:12:14.250 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.70680.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.70680.log.gz 2024-09-14T09:12:14.251 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.62853.log 2024-09-14T09:12:14.255 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.52053.log: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.71371.log.gz 2024-09-14T09:12:14.256 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55401.log 2024-09-14T09:12:14.256 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.62853.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.62853.log.gz 2024-09-14T09:12:14.256 INFO:teuthology.orchestra.run.smithi164.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.46992.log.gz 2024-09-14T09:12:14.257 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51446.log 2024-09-14T09:12:14.258 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.55401.log: 93.3% -- replaced with /var/log/ceph/cluster2-client.mirror.52053.log.gz 2024-09-14T09:12:14.258 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44136.log 2024-09-14T09:12:14.259 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.51446.log: 92.4% -- replaced with /var/log/ceph/cluster2-client.mirror.55401.log.gz 2024-09-14T09:12:14.261 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.36128.log 2024-09-14T09:12:14.261 INFO:teuthology.orchestra.run.smithi164.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.51446.log.gz/var/log/ceph/cluster2-client.mirror.44136.log: 2024-09-14T09:12:14.262 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47783.log 2024-09-14T09:12:14.262 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.admin.36128.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.36128.log.gz 2024-09-14T09:12:14.262 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48741.log 2024-09-14T09:12:14.263 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.47783.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.47783.log.gz 2024-09-14T09:12:14.263 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67044.log 2024-09-14T09:12:14.263 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.48741.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.48741.log.gz 2024-09-14T09:12:14.268 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67922.log 2024-09-14T09:12:14.268 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.67044.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.67044.log.gz 2024-09-14T09:12:14.268 INFO:teuthology.orchestra.run.smithi164.stderr: 93.4% -- replaced with /var/log/ceph/cluster2-client.mirror.44136.log.gz 2024-09-14T09:12:14.268 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42253.log 2024-09-14T09:12:14.268 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose/var/log/ceph/cluster1-client.mirror.67922.log: -- /var/log/ceph/cluster2-client.mirror.44992.log 2024-09-14T09:12:14.268 INFO:teuthology.orchestra.run.smithi164.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.67922.log.gz 2024-09-14T09:12:14.268 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.42253.log: 74.7% -- replaced with /var/log/ceph/cluster2-client.mirror.42253.log.gz 2024-09-14T09:12:14.268 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49590.log 2024-09-14T09:12:14.268 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.44992.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65343.log 2024-09-14T09:12:14.268 INFO:teuthology.orchestra.run.smithi164.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.44992.log.gz 2024-09-14T09:12:14.268 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.49590.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48102.log 2024-09-14T09:12:14.269 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.65343.log: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.65343.log.gz 2024-09-14T09:12:14.277 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71748.log 2024-09-14T09:12:14.292 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.48102.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44566.log 2024-09-14T09:12:14.294 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.71748.log: 90.3% -- replaced with /var/log/ceph/cluster2-client.mirror.71748.log.gz 2024-09-14T09:12:14.296 INFO:teuthology.orchestra.run.smithi164.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.48102.log.gz 2024-09-14T09:12:14.297 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65922.log 2024-09-14T09:12:14.303 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.44566.log: 93.4% -- replaced with /var/log/ceph/cluster2-client.mirror.44566.log.gz 2024-09-14T09:12:14.303 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59706.log 2024-09-14T09:12:14.304 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.65922.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.65922.log.gz 2024-09-14T09:12:14.305 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50633.log 2024-09-14T09:12:14.306 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.59706.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.59706.log.gz 2024-09-14T09:12:14.306 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50823.log 2024-09-14T09:12:14.308 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.50633.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.50633.log.gz 2024-09-14T09:12:14.308 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.62696.log 2024-09-14T09:12:14.309 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.50823.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.50823.log.gz 2024-09-14T09:12:14.309 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55601.log 2024-09-14T09:12:14.310 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.62696.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.62696.log.gz 2024-09-14T09:12:14.310 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.39434.log 2024-09-14T09:12:14.312 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.55601.log: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.55601.log.gz 2024-09-14T09:12:14.312 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64426.log 2024-09-14T09:12:14.314 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.3.39434.log: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.40447.log.gz 2024-09-14T09:12:14.315 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.36338.log 2024-09-14T09:12:14.319 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.64426.log: 93.0% -- replaced with /var/log/ceph/cluster1-client.mirror.3.39434.log.gz 2024-09-14T09:12:14.319 INFO:teuthology.orchestra.run.smithi164.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.64426.log.gz 2024-09-14T09:12:14.319 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63472.log 2024-09-14T09:12:14.319 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose/var/log/ceph/cluster2-client.admin.36338.log: -- /var/log/ceph/cluster2-client.mirror.51848.log 2024-09-14T09:12:14.319 INFO:teuthology.orchestra.run.smithi164.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.36338.log.gz 2024-09-14T09:12:14.319 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.63472.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68690.log 2024-09-14T09:12:14.319 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.51848.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.63472.log.gz 2024-09-14T09:12:14.319 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69205.log 2024-09-14T09:12:14.319 INFO:teuthology.orchestra.run.smithi164.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.51848.log.gz 2024-09-14T09:12:14.319 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.68690.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70081.log 2024-09-14T09:12:14.319 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.69205.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.68690.log.gz 2024-09-14T09:12:14.319 INFO:teuthology.orchestra.run.smithi164.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.69205.log.gz 2024-09-14T09:12:14.320 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41739.log 2024-09-14T09:12:14.324 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose --/var/log/ceph/cluster2-client.mirror.70081.log: /var/log/ceph/cluster2-client.mirror.73121.log 2024-09-14T09:12:14.324 INFO:teuthology.orchestra.run.smithi164.stderr: 79.7% -- replaced with /var/log/ceph/cluster2-client.mirror.70081.log.gz 2024-09-14T09:12:14.324 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.41739.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68372.log 2024-09-14T09:12:14.324 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.73121.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.73121.log.gz 2024-09-14T09:12:14.324 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42968.log 2024-09-14T09:12:14.324 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.68372.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.68372.log.gz 2024-09-14T09:12:14.324 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51631.log 2024-09-14T09:12:14.324 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.42968.log: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.42968.log.gz 2024-09-14T09:12:14.324 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70914.log 2024-09-14T09:12:14.327 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.51631.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.51631.log.gz 2024-09-14T09:12:14.327 INFO:teuthology.orchestra.run.smithi164.stderr: 93.4%gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43454.log 2024-09-14T09:12:14.327 INFO:teuthology.orchestra.run.smithi164.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.41739.log.gz 2024-09-14T09:12:14.327 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53671.log 2024-09-14T09:12:14.327 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.70914.log: /var/log/ceph/cluster1-client.mirror.43454.log: 87.0% -- replaced with /var/log/ceph/cluster2-client.mirror.70914.log.gz 2024-09-14T09:12:14.327 INFO:teuthology.orchestra.run.smithi164.stderr: 89.3% -- replaced with /var/log/ceph/cluster1-client.mirror.43454.log.gz 2024-09-14T09:12:14.327 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37484.log 2024-09-14T09:12:14.327 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.36296.log 2024-09-14T09:12:14.330 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.53671.log: /var/log/ceph/cluster1-client.mirror.37484.log: 54.2% 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.53671.log.gz 2024-09-14T09:12:14.330 INFO:teuthology.orchestra.run.smithi164.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.37484.log.gz 2024-09-14T09:12:14.330 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59664.log 2024-09-14T09:12:14.330 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose --/var/log/ceph/cluster2-client.admin.36296.log: /var/log/ceph/cluster2-client.mirror.41200.log 2024-09-14T09:12:14.330 INFO:teuthology.orchestra.run.smithi164.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.36296.log.gz 2024-09-14T09:12:14.330 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.59664.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52666.log 2024-09-14T09:12:14.331 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.41200.log: 92.0% -- replaced with /var/log/ceph/cluster2-client.mirror.59664.log.gz 2024-09-14T09:12:14.331 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45992.log 2024-09-14T09:12:14.332 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.52666.log: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.52666.log.gz 2024-09-14T09:12:14.333 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48563.log 2024-09-14T09:12:14.333 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.45992.log: 79.8% -- replaced with /var/log/ceph/cluster2-client.mirror.45992.log.gz 2024-09-14T09:12:14.334 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50257.log 2024-09-14T09:12:14.334 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.48563.log: 93.4% -- replaced with /var/log/ceph/cluster2-client.mirror.41200.log.gz 2024-09-14T09:12:14.336 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48483.log 2024-09-14T09:12:14.336 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.50257.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.48563.log.gz 2024-09-14T09:12:14.336 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65757.log 2024-09-14T09:12:14.336 INFO:teuthology.orchestra.run.smithi164.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.50257.log.gz 2024-09-14T09:12:14.336 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.48483.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40571.log 2024-09-14T09:12:14.340 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.65757.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.48483.log.gz 2024-09-14T09:12:14.340 INFO:teuthology.orchestra.run.smithi164.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.65757.log.gz 2024-09-14T09:12:14.341 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64509.log 2024-09-14T09:12:14.341 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.40571.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70894.log 2024-09-14T09:12:14.341 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.64509.log: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.40571.log.gz 2024-09-14T09:12:14.341 INFO:teuthology.orchestra.run.smithi164.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.64509.log.gz 2024-09-14T09:12:14.341 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47863.log 2024-09-14T09:12:14.341 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.70894.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.42770.log 2024-09-14T09:12:14.341 INFO:teuthology.orchestra.run.smithi164.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.70894.log.gz 2024-09-14T09:12:14.341 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.47863.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69683.log 2024-09-14T09:12:14.341 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.admin.42770.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.47863.log.gz 2024-09-14T09:12:14.341 INFO:teuthology.orchestra.run.smithi164.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.admin.42770.log.gz 2024-09-14T09:12:14.341 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73204.log 2024-09-14T09:12:14.344 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54585.log 2024-09-14T09:12:14.344 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.69683.log: /var/log/ceph/cluster2-client.mirror.73204.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.73204.log.gz 2024-09-14T09:12:14.344 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.42253.log 2024-09-14T09:12:14.344 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.54585.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.54585.log.gz 2024-09-14T09:12:14.344 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.1.52215.log 2024-09-14T09:12:14.348 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.0.42253.log: 93.2% -- replaced with /var/log/ceph/cluster1-client.mirror.0.42253.log.gz 2024-09-14T09:12:14.348 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63740.log 2024-09-14T09:12:14.358 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.1.52215.log: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.47905.log.gz 2024-09-14T09:12:14.359 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65901.log 2024-09-14T09:12:14.360 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.63740.log: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.49352.log.gz 2024-09-14T09:12:14.361 INFO:teuthology.orchestra.run.smithi164.stderr: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.63740.log.gz 2024-09-14T09:12:14.362 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65860.log 2024-09-14T09:12:14.362 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose/var/log/ceph/cluster2-client.mirror.65901.log: -- /var/log/ceph/cluster2-client.mirror.49171.log 2024-09-14T09:12:14.362 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.65860.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.65860.log.gz 2024-09-14T09:12:14.367 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60913.log 2024-09-14T09:12:14.367 INFO:teuthology.orchestra.run.smithi164.stderr: 90.1% -- replaced with /var/log/ceph/cluster2-client.mirror.65901.log.gz 2024-09-14T09:12:14.367 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.49171.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53023.log 2024-09-14T09:12:14.367 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.60913.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.49171.log.gz 2024-09-14T09:12:14.367 INFO:teuthology.orchestra.run.smithi164.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.60913.log.gz 2024-09-14T09:12:14.367 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47234.log 2024-09-14T09:12:14.367 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose --/var/log/ceph/cluster1-client.mirror.53023.log: /var/log/ceph/cluster1-client.mirror.56461.log 2024-09-14T09:12:14.367 INFO:teuthology.orchestra.run.smithi164.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.53023.log.gz 2024-09-14T09:12:14.367 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.47234.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.71480.log 2024-09-14T09:12:14.368 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.56461.log: 87.3% -- replaced with /var/log/ceph/cluster2-client.mirror.47234.log.gz 2024-09-14T09:12:14.368 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43941.log 2024-09-14T09:12:14.368 INFO:teuthology.orchestra.run.smithi164.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.56461.log.gz 2024-09-14T09:12:14.368 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.71480.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.71480.log.gz 2024-09-14T09:12:14.368 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46072.log 2024-09-14T09:12:14.371 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51747.log 2024-09-14T09:12:14.371 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.43941.log: /var/log/ceph/cluster2-client.mirror.46072.log: 79.5% -- replaced with /var/log/ceph/cluster2-client.mirror.46072.log.gz 2024-09-14T09:12:14.371 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65302.log 2024-09-14T09:12:14.371 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.51747.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.51747.log.gz 2024-09-14T09:12:14.371 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69378.log 2024-09-14T09:12:14.371 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.65302.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.65302.log.gz 2024-09-14T09:12:14.372 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45949.log 2024-09-14T09:12:14.374 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.69378.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.69378.log.gz 2024-09-14T09:12:14.374 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43382.log 2024-09-14T09:12:14.375 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.45949.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.45949.log.gz 2024-09-14T09:12:14.375 INFO:teuthology.orchestra.run.smithi164.stderr: 93.3% -- replaced with /var/log/ceph/cluster1-client.mirror.43941.log.gz 2024-09-14T09:12:14.375 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51093.log 2024-09-14T09:12:14.375 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose --/var/log/ceph/cluster1-client.mirror.43382.log: /var/log/ceph/cluster2-client.mirror.71522.log 2024-09-14T09:12:14.375 INFO:teuthology.orchestra.run.smithi164.stderr: 89.4% -- replaced with /var/log/ceph/cluster1-client.mirror.43382.log.gz 2024-09-14T09:12:14.375 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.51093.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50799.log 2024-09-14T09:12:14.383 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.71522.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.71522.log.gz 2024-09-14T09:12:14.383 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56678.log 2024-09-14T09:12:14.383 INFO:teuthology.orchestra.run.smithi164.stderr: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.51093.log.gz 2024-09-14T09:12:14.383 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/ceph.tmp-client.admin.27843.log 2024-09-14T09:12:14.383 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.50799.log: /var/log/ceph/cluster1-client.mirror.56678.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.50799.log.gz 2024-09-14T09:12:14.383 INFO:teuthology.orchestra.run.smithi164.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.56678.log.gz 2024-09-14T09:12:14.383 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64133.log 2024-09-14T09:12:14.383 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose --/var/log/ceph/ceph.tmp-client.admin.27843.log: /var/log/ceph/cluster2-client.mirror.53860.log 2024-09-14T09:12:14.383 INFO:teuthology.orchestra.run.smithi164.stderr: 0.0% -- replaced with /var/log/ceph/ceph.tmp-client.admin.27843.log.gz 2024-09-14T09:12:14.383 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.64133.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64195.log 2024-09-14T09:12:14.384 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.53860.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.64133.log.gz 2024-09-14T09:12:14.384 INFO:teuthology.orchestra.run.smithi164.stderr: 87.3% -- replaced with /var/log/ceph/cluster2-client.mirror.53860.log.gz 2024-09-14T09:12:14.384 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43654.log 2024-09-14T09:12:14.384 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose/var/log/ceph/cluster2-client.mirror.64195.log: -- /var/log/ceph/cluster2-client.mirror.72520.log 2024-09-14T09:12:14.384 INFO:teuthology.orchestra.run.smithi164.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.64195.log.gz 2024-09-14T09:12:14.384 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.43654.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43608.log 2024-09-14T09:12:14.384 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.72520.log: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.43654.log.gz 2024-09-14T09:12:14.384 INFO:teuthology.orchestra.run.smithi164.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.72520.log.gz 2024-09-14T09:12:14.384 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47686.log 2024-09-14T09:12:14.384 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose/var/log/ceph/cluster2-client.mirror.43608.log: -- /var/log/ceph/cluster1-client.mirror.44328.log 2024-09-14T09:12:14.384 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.47686.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.43608.log.gz 2024-09-14T09:12:14.384 INFO:teuthology.orchestra.run.smithi164.stderr: 79.7% -- replaced with /var/log/ceph/cluster2-client.mirror.47686.log.gz 2024-09-14T09:12:14.384 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66707.log 2024-09-14T09:12:14.385 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67809.log 2024-09-14T09:12:14.396 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.44328.log: /var/log/ceph/cluster1-client.mirror.66707.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.66707.log.gz 2024-09-14T09:12:14.396 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66617.log 2024-09-14T09:12:14.396 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.67809.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.67809.log.gz 2024-09-14T09:12:14.396 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54043.log 2024-09-14T09:12:14.396 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.66617.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.66617.log.gz 2024-09-14T09:12:14.396 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- 94.1% -- replaced with /var/log/ceph/cluster2-client.mirror.1.52215.log.gz 2024-09-14T09:12:14.396 INFO:teuthology.orchestra.run.smithi164.stderr: /var/log/ceph/cluster2-client.mirror.49373.log 2024-09-14T09:12:14.396 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.54043.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46853.log 2024-09-14T09:12:14.397 INFO:teuthology.orchestra.run.smithi164.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.54043.log.gz/var/log/ceph/cluster2-client.mirror.49373.log: 2024-09-14T09:12:14.397 INFO:teuthology.orchestra.run.smithi164.stderr: 79.8% -- replaced with /var/log/ceph/cluster2-client.mirror.49373.log.gz 2024-09-14T09:12:14.397 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54001.log 2024-09-14T09:12:14.397 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.46853.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49511.log 2024-09-14T09:12:14.397 INFO:teuthology.orchestra.run.smithi164.stderr: 79.7% -- replaced with /var/log/ceph/cluster2-client.mirror.46853.log.gz 2024-09-14T09:12:14.397 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.54001.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45012.log 2024-09-14T09:12:14.397 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.49511.log: 93.4% -- replaced with /var/log/ceph/cluster1-client.mirror.44328.log.gz 2024-09-14T09:12:14.397 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42948.log 2024-09-14T09:12:14.397 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.45012.log: 87.6% 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.54001.log.gz 2024-09-14T09:12:14.397 INFO:teuthology.orchestra.run.smithi164.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.45012.log.gz 2024-09-14T09:12:14.397 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53095.log 2024-09-14T09:12:14.397 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.42948.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72561.log 2024-09-14T09:12:14.397 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.53095.log: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.42948.log.gz 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.53095.log.gz 2024-09-14T09:12:14.397 INFO:teuthology.orchestra.run.smithi164.stderr: 2024-09-14T09:12:14.398 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.36168.log 2024-09-14T09:12:14.398 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.72561.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40093.log 2024-09-14T09:12:14.398 INFO:teuthology.orchestra.run.smithi164.stderr: 86.9% -- replaced with /var/log/ceph/cluster2-client.mirror.72561.log.gz 2024-09-14T09:12:14.408 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.admin.36168.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.36168.log.gz 2024-09-14T09:12:14.408 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59912.log 2024-09-14T09:12:14.408 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.40093.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57174.log 2024-09-14T09:12:14.408 INFO:teuthology.orchestra.run.smithi164.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.40093.log.gz 2024-09-14T09:12:14.408 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.59912.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70438.log 2024-09-14T09:12:14.409 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.57174.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.59912.log.gz 2024-09-14T09:12:14.409 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.39958.log 2024-09-14T09:12:14.409 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.70438.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.70438.log.gz 2024-09-14T09:12:14.409 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63410.log 2024-09-14T09:12:14.409 INFO:teuthology.orchestra.run.smithi164.stderr: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.57174.log.gz 2024-09-14T09:12:14.409 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.39958.log: gzip -5 --verbose -- /var/log/ceph/cluster2-mon.a.log 2024-09-14T09:12:14.409 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.63410.log: 90.7% -- replaced with /var/log/ceph/cluster1-client.mirror.39958.log.gz 2024-09-14T09:12:14.409 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56526.log 2024-09-14T09:12:14.409 INFO:teuthology.orchestra.run.smithi164.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.63410.log.gz 2024-09-14T09:12:14.409 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-mon.a.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51494.log 2024-09-14T09:12:14.409 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.56526.log: 92.0% -- replaced with /var/log/ceph/cluster1-client.mirror.56526.log.gz 2024-09-14T09:12:14.409 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63616.log 2024-09-14T09:12:14.410 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.51494.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.51494.log.gz 2024-09-14T09:12:14.410 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54404.log 2024-09-14T09:12:14.410 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.63616.log: 90.1% -- replaced with /var/log/ceph/cluster2-client.mirror.63616.log.gz 2024-09-14T09:12:14.410 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60446.log 2024-09-14T09:12:14.410 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.54404.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.54404.log.gz 2024-09-14T09:12:14.410 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.71872.log 2024-09-14T09:12:14.410 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.60446.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.60446.log.gz 2024-09-14T09:12:14.410 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57254.log 2024-09-14T09:12:14.410 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.71872.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.71872.log.gz 2024-09-14T09:12:14.410 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58905.log 2024-09-14T09:12:14.413 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.57254.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.57254.log.gz 2024-09-14T09:12:14.413 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47404.log 2024-09-14T09:12:14.413 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.58905.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.58905.log.gz 2024-09-14T09:12:14.414 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44265.log 2024-09-14T09:12:14.414 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.47404.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.47404.log.gz 2024-09-14T09:12:14.414 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44952.log 2024-09-14T09:12:14.417 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.44265.log: 93.4% -- replaced with /var/log/ceph/cluster2-client.mirror.44265.log.gz 2024-09-14T09:12:14.418 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53244.log 2024-09-14T09:12:14.420 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.44952.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.44952.log.gz 2024-09-14T09:12:14.420 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50430.log 2024-09-14T09:12:14.421 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.53244.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.53244.log.gz 2024-09-14T09:12:14.421 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.1.68971.log 2024-09-14T09:12:14.422 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.50430.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.50430.log.gz 2024-09-14T09:12:14.423 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57921.log 2024-09-14T09:12:14.425 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.1.68971.log: 92.4% -- replaced with /var/log/ceph/cluster2-client.mirror.1.68971.log.gz 2024-09-14T09:12:14.426 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49855.log 2024-09-14T09:12:14.427 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.57921.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.57921.log.gz 2024-09-14T09:12:14.427 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.72201.log 2024-09-14T09:12:14.448 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.49855.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57391.log 2024-09-14T09:12:14.451 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.2.72201.log: 92.6% -- replaced with /var/log/ceph/cluster1-client.mirror.2.72201.log.gz 2024-09-14T09:12:14.454 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51964.log 2024-09-14T09:12:14.468 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.57391.log: 92.4% -- replaced with /var/log/ceph/cluster2-client.mirror.57391.log.gz 2024-09-14T09:12:14.477 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67584.log 2024-09-14T09:12:14.477 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.51964.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.51964.log.gz 2024-09-14T09:12:14.482 INFO:teuthology.orchestra.run.smithi164.stderr: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.49590.log.gz 2024-09-14T09:12:14.484 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60551.log 2024-09-14T09:12:14.484 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.67584.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.67584.log.gz 2024-09-14T09:12:14.485 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44732.log 2024-09-14T09:12:14.485 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.60551.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.60551.log.gz 2024-09-14T09:12:14.486 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.37324.log 2024-09-14T09:12:14.487 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.44732.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.44732.log.gz 2024-09-14T09:12:14.487 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40591.log 2024-09-14T09:12:14.489 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.admin.37324.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.37324.log.gz 2024-09-14T09:12:14.489 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67899.log 2024-09-14T09:12:14.490 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.40591.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.40591.log.gz 2024-09-14T09:12:14.490 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70788.log 2024-09-14T09:12:14.490 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.67899.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.67899.log.gz 2024-09-14T09:12:14.492 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49611.log 2024-09-14T09:12:14.492 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.70788.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.70788.log.gz 2024-09-14T09:12:14.492 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64868.log 2024-09-14T09:12:14.493 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.49611.log: 79.8% -- replaced with /var/log/ceph/cluster2-client.mirror.49611.log.gz 2024-09-14T09:12:14.494 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41264.log 2024-09-14T09:12:14.495 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.64868.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.64868.log.gz 2024-09-14T09:12:14.495 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63182.log 2024-09-14T09:12:14.506 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.41264.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51940.log 2024-09-14T09:12:14.508 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.63182.log: 90.1% -- replaced with /var/log/ceph/cluster2-client.mirror.63182.log.gz 2024-09-14T09:12:14.521 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57322.log 2024-09-14T09:12:14.523 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.51940.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.51940.log.gz 2024-09-14T09:12:14.523 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42083.log 2024-09-14T09:12:14.529 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.57322.log: 87.4%gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54428.log 2024-09-14T09:12:14.529 INFO:teuthology.orchestra.run.smithi164.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.57322.log.gz 2024-09-14T09:12:14.530 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.42083.log: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.42083.log.gz 2024-09-14T09:12:14.544 INFO:teuthology.orchestra.run.smithi164.stderr: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.48922.log.gz 2024-09-14T09:12:14.544 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43746.log 2024-09-14T09:12:14.545 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.54428.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.54428.log.gz 2024-09-14T09:12:14.545 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58837.log 2024-09-14T09:12:14.546 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.43746.log: 89.4% -- replaced with /var/log/ceph/cluster1-client.mirror.43746.log.gz 2024-09-14T09:12:14.547 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55308.log 2024-09-14T09:12:14.549 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.58837.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.58837.log.gz 2024-09-14T09:12:14.549 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60023.log 2024-09-14T09:12:14.550 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.55308.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.55308.log.gz 2024-09-14T09:12:14.550 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.36042.log 2024-09-14T09:12:14.551 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.60023.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.60023.log.gz 2024-09-14T09:12:14.551 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45542.log 2024-09-14T09:12:14.553 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.admin.36042.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.36042.log.gz 2024-09-14T09:12:14.553 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52842.log 2024-09-14T09:12:14.554 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.45542.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.45542.log.gz 2024-09-14T09:12:14.554 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49628.log 2024-09-14T09:12:14.556 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.52842.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.52842.log.gz 2024-09-14T09:12:14.556 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67764.log 2024-09-14T09:12:14.557 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.49628.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.49628.log.gz 2024-09-14T09:12:14.557 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57608.log 2024-09-14T09:12:14.558 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.67764.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.67764.log.gz 2024-09-14T09:12:14.558 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53385.log 2024-09-14T09:12:14.561 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.57608.log: 92.4% -- replaced with /var/log/ceph/cluster2-client.mirror.57608.log.gz 2024-09-14T09:12:14.561 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39569.log 2024-09-14T09:12:14.563 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.53385.log: 92.2% -- replaced with /var/log/ceph/cluster2-client.mirror.53385.log.gz 2024-09-14T09:12:14.563 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72348.log 2024-09-14T09:12:14.564 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.39569.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.39569.log.gz 2024-09-14T09:12:14.565 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65199.log 2024-09-14T09:12:14.566 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.72348.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.72348.log.gz 2024-09-14T09:12:14.567 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57879.log 2024-09-14T09:12:14.568 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.65199.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.65199.log.gz 2024-09-14T09:12:14.568 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73746.log 2024-09-14T09:12:14.569 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.57879.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.57879.log.gz 2024-09-14T09:12:14.569 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.74572.log 2024-09-14T09:12:14.572 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.73746.log: 87.0% -- replaced with /var/log/ceph/cluster2-client.mirror.73746.log.gz 2024-09-14T09:12:14.573 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65529.log 2024-09-14T09:12:14.573 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.admin.74572.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.74572.log.gz 2024-09-14T09:12:14.573 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63141.log 2024-09-14T09:12:14.574 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.65529.log: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.65529.log.gz 2024-09-14T09:12:14.574 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65447.log 2024-09-14T09:12:14.575 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.63141.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.63141.log.gz 2024-09-14T09:12:14.576 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63306.log 2024-09-14T09:12:14.577 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.65447.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.65447.log.gz 2024-09-14T09:12:14.577 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44285.log 2024-09-14T09:12:14.579 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.63306.log: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.63306.log.gz 2024-09-14T09:12:14.579 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70643.log 2024-09-14T09:12:14.588 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.44285.log: 93.4% -- replaced with /var/log/ceph/cluster1-client.mirror.44285.log.gz 2024-09-14T09:12:14.588 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72102.log 2024-09-14T09:12:14.588 INFO:teuthology.orchestra.run.smithi164.stderr: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.49511.log.gz 2024-09-14T09:12:14.588 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.70643.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39235.log 2024-09-14T09:12:14.588 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.72102.log: 79.6% -- replaced with /var/log/ceph/cluster2-client.mirror.70643.log.gz 2024-09-14T09:12:14.588 INFO:teuthology.orchestra.run.smithi164.stderr: 87.7%gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66729.log 2024-09-14T09:12:14.588 INFO:teuthology.orchestra.run.smithi164.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.72102.log.gz 2024-09-14T09:12:14.588 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45601.log 2024-09-14T09:12:14.588 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.39235.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48704.log 2024-09-14T09:12:14.591 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.45601.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.45601.log.gz 2024-09-14T09:12:14.591 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70332.log 2024-09-14T09:12:14.591 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.48704.log: 79.8% -- replaced with /var/log/ceph/cluster2-client.mirror.48704.log.gz 2024-09-14T09:12:14.591 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40234.log 2024-09-14T09:12:14.591 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.70332.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.70332.log.gz 2024-09-14T09:12:14.592 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51024.log 2024-09-14T09:12:14.596 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.40234.log: /var/log/ceph/cluster1-client.mirror.66729.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.66729.log.gz 2024-09-14T09:12:14.597 INFO:teuthology.orchestra.run.smithi164.stderr: 93.3% -- replaced with /var/log/ceph/cluster2-client.mirror.40234.log.gz 2024-09-14T09:12:14.601 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44586.log 2024-09-14T09:12:14.601 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.51024.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.51024.log.gz 2024-09-14T09:12:14.601 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61805.log 2024-09-14T09:12:14.604 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.44586.log: 93.4% -- replaced with /var/log/ceph/cluster1-client.mirror.44586.log.gz 2024-09-14T09:12:14.604 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66482.log 2024-09-14T09:12:14.606 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.61805.log: 92.2% -- replaced with /var/log/ceph/cluster1-client.mirror.61805.log.gz 2024-09-14T09:12:14.606 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-osd.1.log 2024-09-14T09:12:14.607 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.66482.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.66482.log.gz 2024-09-14T09:12:14.607 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64112.log 2024-09-14T09:12:14.620 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-osd.1.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61066.log 2024-09-14T09:12:14.622 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.64112.log: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.64112.log.gz 2024-09-14T09:12:14.634 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60978.log 2024-09-14T09:12:14.635 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.61066.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.61066.log.gz 2024-09-14T09:12:14.635 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63100.log 2024-09-14T09:12:14.637 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.60978.log: 92.0% -- replaced with /var/log/ceph/cluster1-client.mirror.60978.log.gz 2024-09-14T09:12:14.650 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63327.log 2024-09-14T09:12:14.651 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.63100.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.63100.log.gz 2024-09-14T09:12:14.664 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73685.log 2024-09-14T09:12:14.664 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.63327.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.63327.log.gz 2024-09-14T09:12:14.679 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41016.log 2024-09-14T09:12:14.679 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.73685.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.73685.log.gz 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.49855.log.gz 2024-09-14T09:12:14.679 INFO:teuthology.orchestra.run.smithi164.stderr: 2024-09-14T09:12:14.679 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64971.log 2024-09-14T09:12:14.680 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.41016.log: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.41016.log.gz 2024-09-14T09:12:14.680 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61985.log 2024-09-14T09:12:14.682 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.64971.log: 90.1% -- replaced with /var/log/ceph/cluster2-client.mirror.64971.log.gz 2024-09-14T09:12:14.682 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.74488.log 2024-09-14T09:12:14.684 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.61985.log: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.61985.log.gz 2024-09-14T09:12:14.684 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.72956.log 2024-09-14T09:12:14.684 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.admin.74488.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.74488.log.gz 2024-09-14T09:12:14.686 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58170.log 2024-09-14T09:12:14.686 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.72956.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.72956.log.gz 2024-09-14T09:12:14.686 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63492.log 2024-09-14T09:12:14.687 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.58170.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.58170.log.gz 2024-09-14T09:12:14.688 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.72434.log 2024-09-14T09:12:14.689 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.63492.log: 90.1% -- replaced with /var/log/ceph/cluster2-client.mirror.63492.log.gz 2024-09-14T09:12:14.691 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60129.log 2024-09-14T09:12:14.691 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.72434.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.72434.log.gz 2024-09-14T09:12:14.692 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42720.log 2024-09-14T09:12:14.693 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.60129.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.60129.log.gz 2024-09-14T09:12:14.693 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42011.log 2024-09-14T09:12:14.706 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.42720.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45912.log 2024-09-14T09:12:14.707 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.42011.log: 88.8% -- replaced with /var/log/ceph/cluster1-client.mirror.42011.log.gz 2024-09-14T09:12:14.721 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53760.log 2024-09-14T09:12:14.721 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.45912.log: 79.7% -- replaced with /var/log/ceph/cluster2-client.mirror.45912.log.gz 2024-09-14T09:12:14.722 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49452.log 2024-09-14T09:12:14.729 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.53760.log: 92.2%gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54246.log 2024-09-14T09:12:14.729 INFO:teuthology.orchestra.run.smithi164.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.53760.log.gz 2024-09-14T09:12:14.738 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.49452.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69460.log 2024-09-14T09:12:14.739 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.54246.log: 79.8% -- replaced with /var/log/ceph/cluster2-client.mirror.49452.log.gz 2024-09-14T09:12:14.740 INFO:teuthology.orchestra.run.smithi164.stderr: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.54246.log.gz 2024-09-14T09:12:14.747 INFO:teuthology.orchestra.run.smithi164.stderr: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.69683.log.gz 2024-09-14T09:12:14.747 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.1.42833.log 2024-09-14T09:12:14.748 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.69460.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.69460.log.gz 2024-09-14T09:12:14.748 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44752.log 2024-09-14T09:12:14.762 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.1.42833.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44414.log 2024-09-14T09:12:14.762 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.44752.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.44752.log.gz 2024-09-14T09:12:14.765 INFO:teuthology.orchestra.run.smithi164.stderr: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.1.42833.log.gz 2024-09-14T09:12:14.765 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55063.log 2024-09-14T09:12:14.781 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.44414.log: 93.4% -- replaced with /var/log/ceph/cluster1-client.mirror.44414.log.gz 2024-09-14T09:12:14.782 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52352.log 2024-09-14T09:12:14.782 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.55063.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.55063.log.gz 2024-09-14T09:12:14.782 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72081.log 2024-09-14T09:12:14.782 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.52352.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.52352.log.gz 2024-09-14T09:12:14.782 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51048.log 2024-09-14T09:12:14.783 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.72081.log: 87.0% -- replaced with /var/log/ceph/cluster2-client.mirror.72081.log.gz 2024-09-14T09:12:14.783 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70955.log 2024-09-14T09:12:14.783 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.51048.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.51048.log.gz 2024-09-14T09:12:14.783 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46051.log 2024-09-14T09:12:14.783 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.70955.log: 87.0% -- replaced with /var/log/ceph/cluster2-client.mirror.70955.log.gz 2024-09-14T09:12:14.784 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.41264.log.gz 2024-09-14T09:12:14.784 INFO:teuthology.orchestra.run.smithi164.stderr: --verbose -- /var/log/ceph/cluster2-client.mirror.44007.log 2024-09-14T09:12:14.784 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44371.log 2024-09-14T09:12:14.786 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.46051.log: /var/log/ceph/cluster2-client.mirror.44007.log: 93.4% -- replaced with /var/log/ceph/cluster2-client.mirror.44007.log.gz 2024-09-14T09:12:14.786 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.62877.log 2024-09-14T09:12:14.814 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.44371.log: 93.4% -- replaced with /var/log/ceph/cluster1-client.mirror.44371.log.gz 2024-09-14T09:12:14.814 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60350.log 2024-09-14T09:12:14.814 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.62877.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.62877.log.gz 2024-09-14T09:12:14.814 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51265.log 2024-09-14T09:12:14.814 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.60350.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.60350.log.gz 2024-09-14T09:12:14.814 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46950.log 2024-09-14T09:12:14.815 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.51265.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.51265.log.gz 2024-09-14T09:12:14.815 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73434.log 2024-09-14T09:12:14.815 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.46950.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.46950.log.gz 2024-09-14T09:12:14.815 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59619.log 2024-09-14T09:12:14.815 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.73434.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.73434.log.gz 2024-09-14T09:12:14.815 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62231.log 2024-09-14T09:12:14.815 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.59619.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.59619.log.gz 2024-09-14T09:12:14.815 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47963.log 2024-09-14T09:12:14.815 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.62231.log: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.62231.log.gz 2024-09-14T09:12:14.816 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45692.log 2024-09-14T09:12:14.816 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.47963.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.47963.log.gz 2024-09-14T09:12:14.816 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57216.log 2024-09-14T09:12:14.816 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.45692.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.45692.log.gz 2024-09-14T09:12:14.816 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.73962.log 2024-09-14T09:12:14.816 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.57216.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.57216.log.gz 2024-09-14T09:12:14.817 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41040.log 2024-09-14T09:12:14.817 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.admin.73962.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.73962.log.gz 2024-09-14T09:12:14.817 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60422.log 2024-09-14T09:12:14.817 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.41040.log: 88.5% -- replaced with /var/log/ceph/cluster1-client.mirror.41040.log.gz 2024-09-14T09:12:14.817 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46563.log 2024-09-14T09:12:14.817 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.60422.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.60422.log.gz 2024-09-14T09:12:14.818 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50209.log 2024-09-14T09:12:14.820 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.46563.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.53510.log 2024-09-14T09:12:14.820 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.50209.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.50209.log.gz 2024-09-14T09:12:14.833 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66842.log 2024-09-14T09:12:14.839 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.0.53510.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67719.log 2024-09-14T09:12:14.847 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.66842.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.35834.log 2024-09-14T09:12:14.847 INFO:teuthology.orchestra.run.smithi164.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.66842.log.gz 2024-09-14T09:12:14.859 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.67719.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.67719.log.gz 2024-09-14T09:12:14.864 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54858.log 2024-09-14T09:12:14.865 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.admin.35834.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.35834.log.gz 2024-09-14T09:12:14.882 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72143.log 2024-09-14T09:12:14.882 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.54858.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.54858.log.gz 2024-09-14T09:12:14.891 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72805.log 2024-09-14T09:12:14.892 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.72143.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.72143.log.gz 2024-09-14T09:12:14.904 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39171.log 2024-09-14T09:12:14.904 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.72805.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.72805.log.gz 2024-09-14T09:12:14.916 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66302.log 2024-09-14T09:12:14.922 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.39171.log: 93.3% -- replaced with /var/log/ceph/cluster2-client.mirror.39171.log.gz 2024-09-14T09:12:14.936 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41583.log 2024-09-14T09:12:14.936 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.66302.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.66302.log.gz 2024-09-14T09:12:14.948 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40972.log 2024-09-14T09:12:14.948 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.41583.log: 88.8% -- replaced with /var/log/ceph/cluster1-client.mirror.41583.log.gz 2024-09-14T09:12:14.959 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44156.log 2024-09-14T09:12:14.961 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.40972.log: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.40972.log.gz 2024-09-14T09:12:14.975 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60733.log 2024-09-14T09:12:14.981 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.44156.log: 93.3% -- replaced with /var/log/ceph/cluster1-client.mirror.44156.log.gz 2024-09-14T09:12:14.984 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53740.log 2024-09-14T09:12:14.999 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.60733.log: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.60733.log.gz 2024-09-14T09:12:14.999 INFO:teuthology.orchestra.run.smithi164.stderr: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.39235.log.gz 2024-09-14T09:12:15.000 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73080.log 2024-09-14T09:12:15.001 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.53740.log: 92.3% -- replaced with /var/log/ceph/cluster2-client.mirror.53740.log.gz 2024-09-14T09:12:15.011 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49310.log 2024-09-14T09:12:15.011 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.73080.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.73080.log.gz 2024-09-14T09:12:15.026 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52328.log 2024-09-14T09:12:15.026 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.49310.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.49310.log.gz 2024-09-14T09:12:15.027 INFO:teuthology.orchestra.run.smithi164.stderr: 92.2% -- replaced with /var/log/ceph/cluster1-client.mirror.0.53510.log.gz 2024-09-14T09:12:15.027 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57941.log 2024-09-14T09:12:15.029 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.52328.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.52328.log.gz 2024-09-14T09:12:15.029 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65385.log 2024-09-14T09:12:15.030 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.57941.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.57941.log.gz 2024-09-14T09:12:15.030 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64992.log 2024-09-14T09:12:15.031 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.65385.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.65385.log.gz 2024-09-14T09:12:15.031 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68102.log 2024-09-14T09:12:15.032 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.64992.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.64992.log.gz 2024-09-14T09:12:15.032 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52746.log 2024-09-14T09:12:15.034 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.68102.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.68102.log.gz 2024-09-14T09:12:15.034 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47197.log 2024-09-14T09:12:15.035 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.52746.log: 87.3% -- replaced with /var/log/ceph/cluster2-client.mirror.52746.log.gz 2024-09-14T09:12:15.035 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68625.log 2024-09-14T09:12:15.036 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.47197.log: 79.6% -- replaced with /var/log/ceph/cluster2-client.mirror.47197.log.gz 2024-09-14T09:12:15.036 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45242.log 2024-09-14T09:12:15.045 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.68625.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70935.log 2024-09-14T09:12:15.053 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.45242.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.45242.log.gz 2024-09-14T09:12:15.058 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47464.log 2024-09-14T09:12:15.058 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.70935.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.70935.log.gz 2024-09-14T09:12:15.062 INFO:teuthology.orchestra.run.smithi164.stderr: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.42720.log.gz 2024-09-14T09:12:15.062 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.43239.log 2024-09-14T09:12:15.063 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.47464.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.47464.log.gz 2024-09-14T09:12:15.065 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64030.log 2024-09-14T09:12:15.065 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.admin.43239.log: 88.1% -- replaced with /var/log/ceph/cluster1-client.admin.43239.log.gz 2024-09-14T09:12:15.065 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49134.log 2024-09-14T09:12:15.067 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.64030.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.64030.log.gz 2024-09-14T09:12:15.067 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.74236.log 2024-09-14T09:12:15.068 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.49134.log: 79.5% -- replaced with /var/log/ceph/cluster2-client.mirror.49134.log.gz 2024-09-14T09:12:15.068 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42472.log 2024-09-14T09:12:15.068 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.admin.74236.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.74236.log.gz 2024-09-14T09:12:15.069 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61740.log 2024-09-14T09:12:15.069 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.42472.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.42472.log.gz 2024-09-14T09:12:15.070 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55927.log 2024-09-14T09:12:15.071 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.61740.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.61740.log.gz 2024-09-14T09:12:15.072 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71979.log 2024-09-14T09:12:15.072 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.55927.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.55927.log.gz 2024-09-14T09:12:15.073 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67854.log 2024-09-14T09:12:15.074 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.71979.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.71979.log.gz 2024-09-14T09:12:15.075 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54930.log 2024-09-14T09:12:15.076 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.67854.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.67854.log.gz 2024-09-14T09:12:15.076 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39795.log 2024-09-14T09:12:15.077 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.54930.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.54930.log.gz 2024-09-14T09:12:15.077 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55264.log 2024-09-14T09:12:15.078 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.39795.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.39795.log.gz 2024-09-14T09:12:15.079 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56606.log 2024-09-14T09:12:15.080 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.55264.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.55264.log.gz 2024-09-14T09:12:15.081 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49004.log 2024-09-14T09:12:15.081 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.56606.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.56606.log.gz 2024-09-14T09:12:15.082 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67089.log 2024-09-14T09:12:15.093 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.49004.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66108.log 2024-09-14T09:12:15.094 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.67089.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.67089.log.gz 2024-09-14T09:12:15.107 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67314.log 2024-09-14T09:12:15.108 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.66108.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.66108.log.gz 2024-09-14T09:12:15.109 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.41392.log 2024-09-14T09:12:15.113 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.67314.log: 86.9% 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.46563.log.gz 2024-09-14T09:12:15.113 INFO:teuthology.orchestra.run.smithi164.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.67314.log.gz 2024-09-14T09:12:15.114 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45813.log 2024-09-14T09:12:15.115 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.admin.41392.log: 87.9% -- replaced with /var/log/ceph/cluster1-client.admin.41392.log.gz 2024-09-14T09:12:15.116 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59981.log 2024-09-14T09:12:15.124 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.45813.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69641.log 2024-09-14T09:12:15.139 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.59981.log: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.59981.log.gz 2024-09-14T09:12:15.146 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64467.log 2024-09-14T09:12:15.147 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.69641.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.69641.log.gz 2024-09-14T09:12:15.157 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63720.log 2024-09-14T09:12:15.159 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.64467.log: 90.2% -- replaced with /var/log/ceph/cluster2-client.mirror.64467.log.gz 2024-09-14T09:12:15.173 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59514.log 2024-09-14T09:12:15.173 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.63720.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.63720.log.gz 2024-09-14T09:12:15.179 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52174.log 2024-09-14T09:12:15.180 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.59514.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.59514.log.gz 2024-09-14T09:12:15.188 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.36907.log 2024-09-14T09:12:15.190 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.52174.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.52174.log.gz 2024-09-14T09:12:15.190 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50350.log 2024-09-14T09:12:15.204 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.2.36907.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69835.log 2024-09-14T09:12:15.205 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.50350.log: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.50350.log.gz 2024-09-14T09:12:15.218 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48842.log 2024-09-14T09:12:15.219 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.69835.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.69835.log.gz 2024-09-14T09:12:15.228 INFO:teuthology.orchestra.run.smithi164.stderr: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.46051.log.gz 2024-09-14T09:12:15.232 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47506.log 2024-09-14T09:12:15.239 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.48842.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59374.log 2024-09-14T09:12:15.246 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.47506.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55087.log 2024-09-14T09:12:15.251 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.59374.log: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.59374.log.gz 2024-09-14T09:12:15.262 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.38820.log 2024-09-14T09:12:15.263 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.55087.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.55087.log.gz 2024-09-14T09:12:15.271 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50143.log 2024-09-14T09:12:15.275 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.0.38820.log: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.68625.log.gz 2024-09-14T09:12:15.286 INFO:teuthology.orchestra.run.smithi164.stderr: 92.7% -- replaced with /var/log/ceph/cluster1-client.mirror.0.38820.log.gz 2024-09-14T09:12:15.288 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.36422.log 2024-09-14T09:12:15.289 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.50143.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.50143.log.gz 2024-09-14T09:12:15.298 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73039.log 2024-09-14T09:12:15.298 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.admin.36422.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.36422.log.gz 2024-09-14T09:12:15.321 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48546.log 2024-09-14T09:12:15.321 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.73039.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.73039.log.gz 2024-09-14T09:12:15.325 INFO:teuthology.orchestra.run.smithi164.stderr: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.45813.log.gz 2024-09-14T09:12:15.326 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54089.log 2024-09-14T09:12:15.327 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.48546.log: 79.5% -- replaced with /var/log/ceph/cluster2-client.mirror.48546.log.gz 2024-09-14T09:12:15.327 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41559.log 2024-09-14T09:12:15.328 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.54089.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.54089.log.gz 2024-09-14T09:12:15.329 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64050.log 2024-09-14T09:12:15.330 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.41559.log: 88.7% -- replaced with /var/log/ceph/cluster1-client.mirror.41559.log.gz 2024-09-14T09:12:15.330 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.39191.log 2024-09-14T09:12:15.332 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.64050.log: 90.1% -- replaced with /var/log/ceph/cluster2-client.mirror.64050.log.gz 2024-09-14T09:12:15.333 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42633.log 2024-09-14T09:12:15.339 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.39191.log: 93.2% -- replaced with /var/log/ceph/cluster1-client.mirror.39191.log.gz 2024-09-14T09:12:15.339 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42211.log 2024-09-14T09:12:15.345 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.42633.log: 93.4% -- replaced with /var/log/ceph/cluster1-client.mirror.42633.log.gz 2024-09-14T09:12:15.345 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.40615.log 2024-09-14T09:12:15.359 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.42211.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70161.log 2024-09-14T09:12:15.360 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.40615.log: 88.0% -- replaced with /var/log/ceph/cluster1-client.mirror.40615.log.gz 2024-09-14T09:12:15.363 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44222.log 2024-09-14T09:12:15.364 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.70161.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.70161.log.gz 2024-09-14T09:12:15.377 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39549.log 2024-09-14T09:12:15.383 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.44222.log: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.44222.log.gz 2024-09-14T09:12:15.396 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61114.log 2024-09-14T09:12:15.397 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.39549.log: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.39549.log.gz 2024-09-14T09:12:15.410 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73891.log 2024-09-14T09:12:15.411 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.61114.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.61114.log.gz 2024-09-14T09:12:15.427 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46707.log 2024-09-14T09:12:15.428 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.73891.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.mirror.73891.log.gz 2024-09-14T09:12:15.434 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69542.log 2024-09-14T09:12:15.435 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.46707.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.46707.log.gz 2024-09-14T09:12:15.440 INFO:teuthology.orchestra.run.smithi164.stderr: 94.0% -- replaced with /var/log/ceph/cluster1-client.mirror.2.36907.log.gz 2024-09-14T09:12:15.446 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69336.log 2024-09-14T09:12:15.446 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.69542.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.69542.log.gz 2024-09-14T09:12:15.456 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70723.log 2024-09-14T09:12:15.456 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.69336.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.69336.log.gz 2024-09-14T09:12:15.475 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40073.log 2024-09-14T09:12:15.476 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.70723.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.70723.log.gz 2024-09-14T09:12:15.483 INFO:teuthology.orchestra.run.smithi164.stderr: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.49004.log.gz 2024-09-14T09:12:15.484 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.74614.log 2024-09-14T09:12:15.487 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.40073.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.40073.log.gz 2024-09-14T09:12:15.487 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64071.log 2024-09-14T09:12:15.487 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.admin.74614.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.74614.log.gz 2024-09-14T09:12:15.487 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73183.log 2024-09-14T09:12:15.488 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.64071.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.64071.log.gz 2024-09-14T09:12:15.489 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47985.log 2024-09-14T09:12:15.490 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.73183.log: 87.1% -- replaced with /var/log/ceph/cluster2-client.mirror.73183.log.gz 2024-09-14T09:12:15.490 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55642.log 2024-09-14T09:12:15.501 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.47985.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57744.log 2024-09-14T09:12:15.503 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.55642.log: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.55642.log.gz 2024-09-14T09:12:15.516 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63761.log 2024-09-14T09:12:15.518 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.57744.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.57744.log.gz 2024-09-14T09:12:15.518 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40314.log 2024-09-14T09:12:15.518 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.63761.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.63761.log.gz 2024-09-14T09:12:15.526 INFO:teuthology.orchestra.run.smithi164.stderr: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.48842.log.gz 2024-09-14T09:12:15.527 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54157.log 2024-09-14T09:12:15.527 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.40314.log: 73.8% -- replaced with /var/log/ceph/cluster2-client.mirror.40314.log.gz 2024-09-14T09:12:15.528 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.40213.log 2024-09-14T09:12:15.529 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.54157.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.54157.log.gz 2024-09-14T09:12:15.529 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47012.log 2024-09-14T09:12:15.529 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.40213.log: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.40213.log.gz 2024-09-14T09:12:15.531 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44070.log 2024-09-14T09:12:15.531 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.47012.log: 79.6% -- replaced with /var/log/ceph/cluster2-client.mirror.47012.log.gz 2024-09-14T09:12:15.531 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.35915.log 2024-09-14T09:12:15.537 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.44070.log: 93.4% -- replaced with /var/log/ceph/cluster1-client.mirror.44070.log.gz 2024-09-14T09:12:15.537 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48880.log 2024-09-14T09:12:15.538 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.admin.35915.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.35915.log.gz 2024-09-14T09:12:15.538 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.1.71182.log 2024-09-14T09:12:15.539 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.48880.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.48880.log.gz 2024-09-14T09:12:15.539 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67067.log 2024-09-14T09:12:15.543 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.1.71182.log: 93.3% -- replaced with /var/log/ceph/cluster2-client.mirror.1.71182.log.gz 2024-09-14T09:12:15.543 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43502.log 2024-09-14T09:12:15.544 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.67067.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.67067.log.gz 2024-09-14T09:12:15.545 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42189.log 2024-09-14T09:12:15.545 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.43502.log: 88.3% -- replaced with /var/log/ceph/cluster1-client.mirror.43502.log.gz 2024-09-14T09:12:15.546 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63926.log 2024-09-14T09:12:15.547 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.42189.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.42189.log.gz 2024-09-14T09:12:15.547 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65323.log 2024-09-14T09:12:15.550 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.63926.log: 90.1% -- replaced with /var/log/ceph/cluster2-client.mirror.63926.log.gz 2024-09-14T09:12:15.550 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47665.log 2024-09-14T09:12:15.550 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.65323.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.65323.log.gz 2024-09-14T09:12:15.551 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54764.log 2024-09-14T09:12:15.563 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.47665.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72705.log 2024-09-14T09:12:15.564 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.54764.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.54764.log.gz 2024-09-14T09:12:15.577 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71728.log 2024-09-14T09:12:15.578 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.72705.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.72705.log.gz 2024-09-14T09:12:15.579 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43358.log 2024-09-14T09:12:15.590 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.71728.log: 87.8%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45146.log 2024-09-14T09:12:15.590 INFO:teuthology.orchestra.run.smithi164.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.71728.log.gz 2024-09-14T09:12:15.591 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.43358.log: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.43358.log.gz 2024-09-14T09:12:15.604 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59864.log 2024-09-14T09:12:15.605 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.45146.log: 79.7% -- replaced with /var/log/ceph/cluster2-client.mirror.45146.log.gz 2024-09-14T09:12:15.607 INFO:teuthology.orchestra.run.smithi164.stderr: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.47506.log.gz 2024-09-14T09:12:15.609 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56212.log 2024-09-14T09:12:15.609 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.59864.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.59864.log.gz 2024-09-14T09:12:15.609 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68484.log 2024-09-14T09:12:15.609 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.56212.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.56212.log.gz 2024-09-14T09:12:15.612 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56835.log 2024-09-14T09:12:15.612 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.68484.log: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.68484.log.gz 2024-09-14T09:12:15.612 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68259.log 2024-09-14T09:12:15.612 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.56835.log: 87.3% -- replaced with /var/log/ceph/cluster2-client.mirror.56835.log.gz 2024-09-14T09:12:15.612 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42452.log 2024-09-14T09:12:15.613 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.68259.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.68259.log.gz 2024-09-14T09:12:15.614 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.74278.log 2024-09-14T09:12:15.614 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.42452.log: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.42452.log.gz 2024-09-14T09:12:15.615 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54517.log 2024-09-14T09:12:15.615 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.admin.74278.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.74278.log.gz 2024-09-14T09:12:15.616 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50281.log 2024-09-14T09:12:15.618 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.54517.log: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.54517.log.gz 2024-09-14T09:12:15.618 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66129.log 2024-09-14T09:12:15.619 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.50281.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.50281.log.gz 2024-09-14T09:12:15.621 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41919.log 2024-09-14T09:12:15.621 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.66129.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.66129.log.gz 2024-09-14T09:12:15.621 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68124.log 2024-09-14T09:12:15.623 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.41919.log: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.41919.log.gz 2024-09-14T09:12:15.623 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64385.log 2024-09-14T09:12:15.624 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.68124.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.68124.log.gz 2024-09-14T09:12:15.624 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59220.log 2024-09-14T09:12:15.626 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.64385.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.64385.log.gz 2024-09-14T09:12:15.626 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54205.log 2024-09-14T09:12:15.638 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.59220.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52012.log 2024-09-14T09:12:15.639 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.54205.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.54205.log.gz 2024-09-14T09:12:15.652 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53119.log 2024-09-14T09:12:15.653 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.52012.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.52012.log.gz 2024-09-14T09:12:15.654 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45734.log 2024-09-14T09:12:15.660 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.53119.log: 87.7%gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50522.log 2024-09-14T09:12:15.661 INFO:teuthology.orchestra.run.smithi164.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.53119.log.gz 2024-09-14T09:12:15.674 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.45734.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70767.log 2024-09-14T09:12:15.675 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.50522.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.50522.log.gz 2024-09-14T09:12:15.685 INFO:teuthology.orchestra.run.smithi164.stderr: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.42211.log.gz 2024-09-14T09:12:15.686 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56413.log 2024-09-14T09:12:15.686 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.70767.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.70767.log.gz 2024-09-14T09:12:15.687 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55885.log 2024-09-14T09:12:15.687 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.56413.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.56413.log.gz 2024-09-14T09:12:15.688 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67202.log 2024-09-14T09:12:15.688 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.55885.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.55885.log.gz 2024-09-14T09:12:15.690 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71124.log 2024-09-14T09:12:15.690 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.67202.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.67202.log.gz 2024-09-14T09:12:15.690 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63513.log 2024-09-14T09:12:15.690 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.71124.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.71124.log.gz 2024-09-14T09:12:15.692 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42656.log 2024-09-14T09:12:15.692 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.63513.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.63513.log.gz 2024-09-14T09:12:15.693 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54742.log 2024-09-14T09:12:15.698 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.42656.log: 93.3% -- replaced with /var/log/ceph/cluster2-client.mirror.42656.log.gz 2024-09-14T09:12:15.699 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49230.log 2024-09-14T09:12:15.700 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.54742.log: 92.2% -- replaced with /var/log/ceph/cluster1-client.mirror.54742.log.gz 2024-09-14T09:12:15.708 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70479.log 2024-09-14T09:12:15.710 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.49230.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.49230.log.gz 2024-09-14T09:12:15.710 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63823.log 2024-09-14T09:12:15.710 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.70479.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.70479.log.gz 2024-09-14T09:12:15.714 INFO:teuthology.orchestra.run.smithi164.stderr: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.59220.log.gz 2024-09-14T09:12:15.714 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55706.log 2024-09-14T09:12:15.715 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.63823.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.63823.log.gz 2024-09-14T09:12:15.716 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.40900.log 2024-09-14T09:12:15.717 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.55706.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.55706.log.gz 2024-09-14T09:12:15.717 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67157.log 2024-09-14T09:12:15.718 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.admin.40900.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.40900.log.gz 2024-09-14T09:12:15.719 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47447.log 2024-09-14T09:12:15.719 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.67157.log: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.67157.log.gz 2024-09-14T09:12:15.720 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66932.log 2024-09-14T09:12:15.720 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.47447.log: 79.5% -- replaced with /var/log/ceph/cluster2-client.mirror.47447.log.gz 2024-09-14T09:12:15.722 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67742.log 2024-09-14T09:12:15.722 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.66932.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.66932.log.gz 2024-09-14T09:12:15.722 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63389.log 2024-09-14T09:12:15.723 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.67742.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.67742.log.gz 2024-09-14T09:12:15.723 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43921.log 2024-09-14T09:12:15.725 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.63389.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.63389.log.gz 2024-09-14T09:12:15.725 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53405.log 2024-09-14T09:12:15.727 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.43921.log: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.47665.log.gz 2024-09-14T09:12:15.728 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67427.log 2024-09-14T09:12:15.729 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.53405.log: 92.1% -- replaced with /var/log/ceph/cluster1-client.mirror.53405.log.gz 2024-09-14T09:12:15.734 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.72891.log 2024-09-14T09:12:15.734 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.67427.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.67427.log.gz 2024-09-14T09:12:15.734 INFO:teuthology.orchestra.run.smithi164.stderr: 93.3%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50392.log 2024-09-14T09:12:15.734 INFO:teuthology.orchestra.run.smithi164.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.43921.log.gz 2024-09-14T09:12:15.735 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.72891.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59538.log 2024-09-14T09:12:15.735 INFO:teuthology.orchestra.run.smithi164.stderr: 87.2% -- replaced with /var/log/ceph/cluster1-client.mirror.72891.log.gz 2024-09-14T09:12:15.735 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.50392.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43194.log 2024-09-14T09:12:15.735 INFO:teuthology.orchestra.run.smithi164.stderr: 88.4%/var/log/ceph/cluster1-client.mirror.59538.log: -- replaced with /var/log/ceph/cluster2-client.mirror.50392.log.gz 2024-09-14T09:12:15.735 INFO:teuthology.orchestra.run.smithi164.stderr: 87.7%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62809.log 2024-09-14T09:12:15.735 INFO:teuthology.orchestra.run.smithi164.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.59538.log.gz 2024-09-14T09:12:15.735 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.43194.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59643.log 2024-09-14T09:12:15.735 INFO:teuthology.orchestra.run.smithi164.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.43194.log.gz 2024-09-14T09:12:15.735 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.62809.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66005.log 2024-09-14T09:12:15.735 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.59643.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.62809.log.gz 2024-09-14T09:12:15.739 INFO:teuthology.orchestra.run.smithi164.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.59643.log.gz 2024-09-14T09:12:15.739 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37729.log 2024-09-14T09:12:15.740 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose --/var/log/ceph/cluster2-client.mirror.66005.log: /var/log/ceph/cluster2-client.admin.36716.log 2024-09-14T09:12:15.740 INFO:teuthology.orchestra.run.smithi164.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.66005.log.gz 2024-09-14T09:12:15.740 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.37729.log: 57.4% -- replaced with /var/log/ceph/cluster2-client.mirror.37729.log.gz 2024-09-14T09:12:15.740 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61090.log 2024-09-14T09:12:15.740 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.admin.36716.log: 0.0%gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61692.log 2024-09-14T09:12:15.740 INFO:teuthology.orchestra.run.smithi164.stderr: -- replaced with /var/log/ceph/cluster2-client.admin.36716.log.gz 2024-09-14T09:12:15.740 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66369.log 2024-09-14T09:12:15.740 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.61090.log: /var/log/ceph/cluster1-client.mirror.61692.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.61090.log.gz 2024-09-14T09:12:15.740 INFO:teuthology.orchestra.run.smithi164.stderr:gzip 87.7% -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64579.log 2024-09-14T09:12:15.740 INFO:teuthology.orchestra.run.smithi164.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.61692.log.gz 2024-09-14T09:12:15.740 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.66369.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.66369.log.gz 2024-09-14T09:12:15.740 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44932.log 2024-09-14T09:12:15.740 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58616.log 2024-09-14T09:12:15.740 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.64579.log: /var/log/ceph/cluster2-client.mirror.44932.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.64579.log.gz 2024-09-14T09:12:15.743 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72825.log 2024-09-14T09:12:15.743 INFO:teuthology.orchestra.run.smithi164.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.44932.log.gz 2024-09-14T09:12:15.743 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.58616.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.58616.log.gz 2024-09-14T09:12:15.743 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57153.log 2024-09-14T09:12:15.744 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.72825.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58556.log 2024-09-14T09:12:15.744 INFO:teuthology.orchestra.run.smithi164.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.72825.log.gz 2024-09-14T09:12:15.744 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.57153.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66087.log 2024-09-14T09:12:15.744 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.58556.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.57153.log.gz 2024-09-14T09:12:15.744 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46809.log 2024-09-14T09:12:15.745 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.66087.log: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.58556.log.gz 2024-09-14T09:12:15.745 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59155.log 2024-09-14T09:12:15.745 INFO:teuthology.orchestra.run.smithi164.stderr: 90.1% -- replaced with /var/log/ceph/cluster2-client.mirror.66087.log.gz 2024-09-14T09:12:15.746 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.46809.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.36837.log 2024-09-14T09:12:15.746 INFO:teuthology.orchestra.run.smithi164.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.46809.log.gz/var/log/ceph/cluster1-client.mirror.59155.log: 2024-09-14T09:12:15.746 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73916.log 2024-09-14T09:12:15.746 INFO:teuthology.orchestra.run.smithi164.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.59155.log.gz 2024-09-14T09:12:15.747 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.0.36837.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.38689.log 2024-09-14T09:12:15.747 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.73916.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.mirror.73916.log.gz 2024-09-14T09:12:15.747 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52818.log 2024-09-14T09:12:15.748 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.admin.38689.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.38689.log.gz 2024-09-14T09:12:15.748 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71806.log 2024-09-14T09:12:15.749 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.52818.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.52818.log.gz 2024-09-14T09:12:15.749 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63637.log 2024-09-14T09:12:15.750 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.71806.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.71806.log.gz 2024-09-14T09:12:15.750 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55947.log 2024-09-14T09:12:15.751 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.63637.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.63637.log.gz 2024-09-14T09:12:15.752 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44629.log 2024-09-14T09:12:15.753 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.55947.log: 87.3% -- replaced with /var/log/ceph/cluster2-client.mirror.55947.log.gz 2024-09-14T09:12:15.753 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65550.log 2024-09-14T09:12:15.755 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.44629.log: 93.4% -- replaced with /var/log/ceph/cluster1-client.mirror.0.36837.log.gz 2024-09-14T09:12:15.755 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42544.log 2024-09-14T09:12:15.756 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.65550.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.65550.log.gz 2024-09-14T09:12:15.756 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42992.log 2024-09-14T09:12:15.757 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.42544.log: 88.8% -- replaced with /var/log/ceph/cluster1-client.mirror.42544.log.gz 2024-09-14T09:12:15.757 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58998.log 2024-09-14T09:12:15.759 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.42992.log: 89.2% -- replaced with /var/log/ceph/cluster1-client.mirror.42992.log.gz 2024-09-14T09:12:15.760 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.40499.log 2024-09-14T09:12:15.760 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.58998.log: 93.4% -- replaced with /var/log/ceph/cluster1-client.mirror.44629.log.gz 2024-09-14T09:12:15.760 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69185.log 2024-09-14T09:12:15.770 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.admin.40499.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.40499.log.gz 2024-09-14T09:12:15.770 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57037.log 2024-09-14T09:12:15.770 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.69185.log: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.58998.log.gz 2024-09-14T09:12:15.770 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72767.log 2024-09-14T09:12:15.770 INFO:teuthology.orchestra.run.smithi164.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.69185.log.gz 2024-09-14T09:12:15.770 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.57037.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59333.log 2024-09-14T09:12:15.770 INFO:teuthology.orchestra.run.smithi164.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.57037.log.gz 2024-09-14T09:12:15.771 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.72767.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42035.log 2024-09-14T09:12:15.771 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.59333.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.59333.log.gz 2024-09-14T09:12:15.771 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52376.log 2024-09-14T09:12:15.771 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.42035.log: 90.3% -- replaced with /var/log/ceph/cluster2-client.mirror.72767.log.gz 2024-09-14T09:12:15.771 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50474.log 2024-09-14T09:12:15.771 INFO:teuthology.orchestra.run.smithi164.stderr: 88.6% -- replaced with /var/log/ceph/cluster1-client.mirror.42035.log.gz 2024-09-14T09:12:15.771 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.52376.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.43266.log 2024-09-14T09:12:15.771 INFO:teuthology.orchestra.run.smithi164.stderr: 87.9%/var/log/ceph/cluster1-client.mirror.50474.log: -- replaced with /var/log/ceph/cluster1-client.mirror.52376.log.gz 2024-09-14T09:12:15.771 INFO:teuthology.orchestra.run.smithi164.stderr: 87.4%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49151.log 2024-09-14T09:12:15.771 INFO:teuthology.orchestra.run.smithi164.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.50474.log.gz 2024-09-14T09:12:15.771 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.admin.43266.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68548.log 2024-09-14T09:12:15.771 INFO:teuthology.orchestra.run.smithi164.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.admin.43266.log.gz 2024-09-14T09:12:15.771 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.49151.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55726.log 2024-09-14T09:12:15.771 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.68548.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.49151.log.gz 2024-09-14T09:12:15.771 INFO:teuthology.orchestra.run.smithi164.stderr: 2.6% -- replaced with /var/log/ceph/cluster2-client.mirror.68548.log.gz 2024-09-14T09:12:15.772 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52417.log 2024-09-14T09:12:15.772 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.55726.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45098.log 2024-09-14T09:12:15.772 INFO:teuthology.orchestra.run.smithi164.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.55726.log.gz 2024-09-14T09:12:15.772 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.52417.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41607.log 2024-09-14T09:12:15.772 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.45098.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.45098.log.gz 2024-09-14T09:12:15.772 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.42743.log 2024-09-14T09:12:15.780 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.41607.log: 92.1% -- replaced with /var/log/ceph/cluster1-client.mirror.52417.log.gz 2024-09-14T09:12:15.780 INFO:teuthology.orchestra.run.smithi164.stderr: 88.7% -- replaced with /var/log/ceph/cluster1-client.mirror.41607.log.gz 2024-09-14T09:12:15.780 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51868.log 2024-09-14T09:12:15.780 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.admin.42743.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69106.log 2024-09-14T09:12:15.781 INFO:teuthology.orchestra.run.smithi164.stderr: 88.2% -- replaced with /var/log/ceph/cluster1-client.admin.42743.log.gz 2024-09-14T09:12:15.781 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.51868.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66504.log 2024-09-14T09:12:15.781 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.69106.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.51868.log.gz 2024-09-14T09:12:15.781 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41088.log 2024-09-14T09:12:15.781 INFO:teuthology.orchestra.run.smithi164.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.69106.log.gz 2024-09-14T09:12:15.781 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.66504.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69085.log 2024-09-14T09:12:15.782 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.41088.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.66504.log.gz 2024-09-14T09:12:15.782 INFO:teuthology.orchestra.run.smithi164.stderr: 88.2%gzip -- replaced with /var/log/ceph/cluster1-client.mirror.41088.log.gz -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46131.log 2024-09-14T09:12:15.782 INFO:teuthology.orchestra.run.smithi164.stderr: 2024-09-14T09:12:15.782 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59570.log 2024-09-14T09:12:15.782 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.69085.log: /var/log/ceph/cluster2-client.mirror.46131.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.69085.log.gz 2024-09-14T09:12:15.782 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44972.log 2024-09-14T09:12:15.782 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.59570.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.59570.log.gz 2024-09-14T09:12:15.783 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44672.log 2024-09-14T09:12:15.783 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.44972.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.44972.log.gz 2024-09-14T09:12:15.783 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64744.log 2024-09-14T09:12:15.783 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.44672.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.44672.log.gz 2024-09-14T09:12:15.783 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46749.log 2024-09-14T09:12:15.785 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.64744.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.64744.log.gz 2024-09-14T09:12:15.785 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.35403.log 2024-09-14T09:12:15.797 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.46749.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59840.log 2024-09-14T09:12:15.797 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.admin.35403.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.35403.log.gz 2024-09-14T09:12:15.803 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.40165.log 2024-09-14T09:12:15.816 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.59840.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.59840.log.gz 2024-09-14T09:12:15.817 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68566.log 2024-09-14T09:12:15.817 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.40165.log: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.40165.log.gz 2024-09-14T09:12:15.838 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47155.log 2024-09-14T09:12:15.838 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.68566.log: 79.6% -- replaced with /var/log/ceph/cluster2-client.mirror.68566.log.gz 2024-09-14T09:12:15.847 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49272.log 2024-09-14T09:12:15.850 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.47155.log: 87.4% 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.47985.log.gz 2024-09-14T09:12:15.851 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56260.log 2024-09-14T09:12:15.858 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.49272.log: 93.6% -- replaced with /var/log/ceph/cluster2-client.mirror.47155.log.gz 2024-09-14T09:12:15.858 INFO:teuthology.orchestra.run.smithi164.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.45734.log.gz 2024-09-14T09:12:15.858 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45390.log 2024-09-14T09:12:15.859 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.56260.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.56260.log.gz 2024-09-14T09:12:15.861 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73726.log 2024-09-14T09:12:15.861 INFO:teuthology.orchestra.run.smithi164.stderr: 90.0% -- replaced with /var/log/ceph/cluster2-mon.a.log.gz 2024-09-14T09:12:15.861 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.45390.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61785.log 2024-09-14T09:12:15.861 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.73726.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.73726.log.gz 2024-09-14T09:12:15.862 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64723.log 2024-09-14T09:12:15.864 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.61785.log: 92.4% -- replaced with /var/log/ceph/cluster2-client.mirror.61785.log.gz 2024-09-14T09:12:15.864 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71828.log 2024-09-14T09:12:15.866 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.64723.log: 90.1% -- replaced with /var/log/ceph/cluster2-client.mirror.64723.log.gz 2024-09-14T09:12:15.866 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46029.log 2024-09-14T09:12:15.887 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.71828.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64703.log 2024-09-14T09:12:15.888 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.46029.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.46029.log.gz 2024-09-14T09:12:15.904 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66170.log 2024-09-14T09:12:15.904 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.64703.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.64703.log.gz 2024-09-14T09:12:15.916 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56654.log 2024-09-14T09:12:15.917 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.66170.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.66170.log.gz 2024-09-14T09:12:15.930 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54605.log 2024-09-14T09:12:15.930 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.56654.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.56654.log.gz 2024-09-14T09:12:15.943 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56586.log 2024-09-14T09:12:15.944 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.54605.log: 87.3% -- replaced with /var/log/ceph/cluster2-client.mirror.54605.log.gz 2024-09-14T09:12:15.952 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69480.log 2024-09-14T09:12:15.962 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.56586.log: 88.9%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46210.log 2024-09-14T09:12:15.966 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.69480.log: 86.9% -- replaced with /var/log/ceph/cluster2-client.mirror.56586.log.gz 2024-09-14T09:12:15.971 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60081.log 2024-09-14T09:12:15.975 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.46210.log: -- replaced with /var/log/ceph/cluster2-client.mirror.69480.log.gz 2024-09-14T09:12:15.980 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70561.log 2024-09-14T09:12:15.993 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.60081.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.60081.log.gz 2024-09-14T09:12:15.998 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.62990.log 2024-09-14T09:12:15.999 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.70561.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.70561.log.gz 2024-09-14T09:12:16.017 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39030.log 2024-09-14T09:12:16.019 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.62990.log: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.62990.log.gz 2024-09-14T09:12:16.030 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43792.log 2024-09-14T09:12:16.031 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.39030.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.39030.log.gz 2024-09-14T09:12:16.031 INFO:teuthology.orchestra.run.smithi164.stderr: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.45390.log.gz 2024-09-14T09:12:16.032 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72020.log 2024-09-14T09:12:16.032 INFO:teuthology.orchestra.run.smithi164.stderr: 93.7%/var/log/ceph/cluster2-client.mirror.43792.log: -- replaced with /var/log/ceph/cluster2-client.mirror.46749.log.gz 2024-09-14T09:12:16.037 INFO:teuthology.orchestra.run.smithi164.stderr: 93.3% -- replaced with /var/log/ceph/cluster2-client.mirror.43792.log.gz 2024-09-14T09:12:16.041 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68531.log 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.46131.log.gz 2024-09-14T09:12:16.041 INFO:teuthology.orchestra.run.smithi164.stderr: 2024-09-14T09:12:16.041 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45416.log 2024-09-14T09:12:16.041 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.72020.log: /var/log/ceph/cluster1-client.mirror.68531.log: 87.7% 1.3% -- replaced with /var/log/ceph/cluster1-client.mirror.68531.log.gz 2024-09-14T09:12:16.041 INFO:teuthology.orchestra.run.smithi164.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.72020.log.gz 2024-09-14T09:12:16.041 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56389.log 2024-09-14T09:12:16.041 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.45416.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44199.log 2024-09-14T09:12:16.041 INFO:teuthology.orchestra.run.smithi164.stderr: 79.7% -- replaced with /var/log/ceph/cluster2-client.mirror.45416.log.gz 2024-09-14T09:12:16.041 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.56389.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56815.log 2024-09-14T09:12:16.041 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.44199.log: 87.3% -- replaced with /var/log/ceph/cluster2-client.mirror.56389.log.gz 2024-09-14T09:12:16.042 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70397.log 2024-09-14T09:12:16.042 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.56815.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.56815.log.gz 2024-09-14T09:12:16.042 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52726.log 2024-09-14T09:12:16.043 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.70397.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.70397.log.gz 2024-09-14T09:12:16.043 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67539.log 2024-09-14T09:12:16.044 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.52726.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.52726.log.gz 2024-09-14T09:12:16.045 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49813.log 2024-09-14T09:12:16.045 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.67539.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.67539.log.gz 2024-09-14T09:12:16.046 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42496.log 2024-09-14T09:12:16.046 INFO:teuthology.orchestra.run.smithi164.stderr: 93.4% -- replaced with /var/log/ceph/cluster1-client.mirror.44199.log.gz 2024-09-14T09:12:16.047 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.49813.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.49813.log.gz 2024-09-14T09:12:16.047 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.1.70242.log 2024-09-14T09:12:16.048 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57491.log 2024-09-14T09:12:16.049 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.42496.log: /var/log/ceph/cluster2-client.mirror.1.70242.log: 89.1% -- replaced with /var/log/ceph/cluster1-client.mirror.42496.log.gz 2024-09-14T09:12:16.049 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55971.log 2024-09-14T09:12:16.056 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.57491.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.57491.log.gz 2024-09-14T09:12:16.056 INFO:teuthology.orchestra.run.smithi164.stderr: 92.0% -- replaced with /var/log/ceph/cluster2-client.mirror.1.70242.log.gz 2024-09-14T09:12:16.056 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66237.log 2024-09-14T09:12:16.056 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose --/var/log/ceph/cluster1-client.mirror.55971.log: /var/log/ceph/cluster1-client.mirror.51518.log 2024-09-14T09:12:16.056 INFO:teuthology.orchestra.run.smithi164.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.55971.log.gz/var/log/ceph/cluster1-client.mirror.66237.log: 2024-09-14T09:12:16.056 INFO:teuthology.orchestra.run.smithi164.stderr: 1.3% -- replaced with /var/log/ceph/cluster1-client.mirror.66237.log.gz 2024-09-14T09:12:16.056 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66437.log 2024-09-14T09:12:16.056 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.51518.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52866.log 2024-09-14T09:12:16.056 INFO:teuthology.orchestra.run.smithi164.stderr: 87.7%/var/log/ceph/cluster1-client.mirror.66437.log: -- replaced with /var/log/ceph/cluster1-client.mirror.51518.log.gz 2024-09-14T09:12:16.056 INFO:teuthology.orchestra.run.smithi164.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.66437.log.gz 2024-09-14T09:12:16.057 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45433.log 2024-09-14T09:12:16.057 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.52866.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.36380.log 2024-09-14T09:12:16.057 INFO:teuthology.orchestra.run.smithi164.stderr: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.52866.log.gz 2024-09-14T09:12:16.057 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.45433.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43172.log 2024-09-14T09:12:16.057 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.admin.36380.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.36380.log.gz 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.45433.log.gz 2024-09-14T09:12:16.057 INFO:teuthology.orchestra.run.smithi164.stderr: 2024-09-14T09:12:16.057 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48230.log 2024-09-14T09:12:16.057 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose/var/log/ceph/cluster1-client.mirror.43172.log: -- /var/log/ceph/cluster2-client.mirror.60306.log 2024-09-14T09:12:16.057 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.48230.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.48230.log.gz 2024-09-14T09:12:16.063 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47606.log 2024-09-14T09:12:16.063 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.60306.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.60306.log.gz 2024-09-14T09:12:16.063 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56064.log 2024-09-14T09:12:16.064 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.47606.log: 79.9% -- replaced with /var/log/ceph/cluster2-client.mirror.47606.log.gz 2024-09-14T09:12:16.064 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68012.log 2024-09-14T09:12:16.064 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.56064.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.43172.log.gz 2024-09-14T09:12:16.064 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54360.log 2024-09-14T09:12:16.064 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.68012.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.68012.log.gz 2024-09-14T09:12:16.064 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2.log 2024-09-14T09:12:16.064 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.54360.log: 92.3% -- replaced with /var/log/ceph/cluster2-client.mirror.56064.log.gz 2024-09-14T09:12:16.064 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40839.log 2024-09-14T09:12:16.064 INFO:teuthology.orchestra.run.smithi164.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.54360.log.gz 2024-09-14T09:12:16.064 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.38535.log 2024-09-14T09:12:16.068 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.40839.log: 87.7% -- replaced with /var/log/ceph/cluster2.log.gz 2024-09-14T09:12:16.068 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45564.log 2024-09-14T09:12:16.068 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.admin.38535.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.38535.log.gz 2024-09-14T09:12:16.069 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65219.log 2024-09-14T09:12:16.085 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.45564.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44792.log 2024-09-14T09:12:16.087 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.65219.log: 90.1% -- replaced with /var/log/ceph/cluster2-client.mirror.65219.log.gz 2024-09-14T09:12:16.104 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62970.log 2024-09-14T09:12:16.104 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.44792.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.44792.log.gz 2024-09-14T09:12:16.112 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55222.log 2024-09-14T09:12:16.122 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.62970.log: 92.5% 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.71828.log.gz 2024-09-14T09:12:16.122 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60509.log 2024-09-14T09:12:16.122 INFO:teuthology.orchestra.run.smithi164.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.62970.log.gz 2024-09-14T09:12:16.122 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.55222.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.39054.log 2024-09-14T09:12:16.123 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.60509.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.60509.log.gz 2024-09-14T09:12:16.123 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.62351.log 2024-09-14T09:12:16.123 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.39054.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.39054.log.gz 2024-09-14T09:12:16.124 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.39593.log 2024-09-14T09:12:16.131 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.2.62351.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.55222.log.gz 2024-09-14T09:12:16.134 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68711.log 2024-09-14T09:12:16.135 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.39593.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.39593.log.gz 2024-09-14T09:12:16.148 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70140.log 2024-09-14T09:12:16.149 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.68711.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.68711.log.gz 2024-09-14T09:12:16.149 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44394.log 2024-09-14T09:12:16.156 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.70140.log: 93.6% -- replaced with /var/log/ceph/cluster2-client.mirror.49272.log.gz 2024-09-14T09:12:16.162 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71958.log 2024-09-14T09:12:16.167 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.44394.log: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.44394.log.gz 2024-09-14T09:12:16.168 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58857.log 2024-09-14T09:12:16.177 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.71958.log: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.46210.log.gz 2024-09-14T09:12:16.177 INFO:teuthology.orchestra.run.smithi164.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.71958.log.gz 2024-09-14T09:12:16.178 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40525.log 2024-09-14T09:12:16.180 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.58857.log: 92.2% -- replaced with /var/log/ceph/cluster1-client.mirror.2.62351.log.gz 2024-09-14T09:12:16.180 INFO:teuthology.orchestra.run.smithi164.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.58857.log.gz 2024-09-14T09:12:16.180 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61869.log 2024-09-14T09:12:16.180 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56907.log 2024-09-14T09:12:16.181 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.40525.log: /var/log/ceph/cluster2-client.mirror.61869.log: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.40525.log.gz 2024-09-14T09:12:16.182 INFO:teuthology.orchestra.run.smithi164.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.61869.log.gz 2024-09-14T09:12:16.182 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71329.log 2024-09-14T09:12:16.183 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.56907.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.56907.log.gz 2024-09-14T09:12:16.183 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.72411.log 2024-09-14T09:12:16.183 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.71329.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72847.log 2024-09-14T09:12:16.184 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.72411.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.71329.log.gz 2024-09-14T09:12:16.184 INFO:teuthology.orchestra.run.smithi164.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.72411.log.gz 2024-09-14T09:12:16.185 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62556.log 2024-09-14T09:12:16.185 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.72847.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68796.log 2024-09-14T09:12:16.186 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.62556.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.62556.log.gz 2024-09-14T09:12:16.186 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47256.log 2024-09-14T09:12:16.186 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.68796.log: 84.6% -- replaced with /var/log/ceph/cluster2-client.mirror.68796.log.gz 2024-09-14T09:12:16.187 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.3.49044.log 2024-09-14T09:12:16.207 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.47256.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53473.log 2024-09-14T09:12:16.213 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.3.49044.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43674.log 2024-09-14T09:12:16.220 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.53473.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58795.log 2024-09-14T09:12:16.221 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.43674.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.53473.log.gz 2024-09-14T09:12:16.232 INFO:teuthology.orchestra.run.smithi164.stderr: 89.4%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40926.log 2024-09-14T09:12:16.232 INFO:teuthology.orchestra.run.smithi164.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.43674.log.gz 2024-09-14T09:12:16.233 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.58795.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.58795.log.gz 2024-09-14T09:12:16.240 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38399.log 2024-09-14T09:12:16.241 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.40926.log: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.40926.log.gz 2024-09-14T09:12:16.249 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.37348.log 2024-09-14T09:12:16.263 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.38399.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42592.log 2024-09-14T09:12:16.263 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.admin.37348.log: 80.3% -- replaced with /var/log/ceph/cluster1-client.admin.37348.log.gz 2024-09-14T09:12:16.275 INFO:teuthology.orchestra.run.smithi164.stderr: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.45564.log.gz 2024-09-14T09:12:16.276 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.36040.log 2024-09-14T09:12:16.276 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.42592.log: 88.6% -- replaced with /var/log/ceph/cluster1-client.mirror.42592.log.gz 2024-09-14T09:12:16.299 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46267.log 2024-09-14T09:12:16.299 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.admin.36040.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.36040.log.gz 2024-09-14T09:12:16.302 INFO:teuthology.orchestra.run.smithi164.stderr: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.3.49044.log.gz 2024-09-14T09:12:16.324 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.46600.log 2024-09-14T09:12:16.325 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.46267.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.46267.log.gz 2024-09-14T09:12:16.332 INFO:teuthology.orchestra.run.smithi164.stderr: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.72847.log.gz 2024-09-14T09:12:16.333 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.36464.log 2024-09-14T09:12:16.347 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.0.46600.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69958.log 2024-09-14T09:12:16.347 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.admin.36464.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.36464.log.gz 2024-09-14T09:12:16.356 INFO:teuthology.orchestra.run.smithi164.stderr: 93.8%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48065.log 2024-09-14T09:12:16.357 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.69958.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.69958.log.gz 2024-09-14T09:12:16.357 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47723.log 2024-09-14T09:12:16.364 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.48065.log: -- replaced with /var/log/ceph/cluster2-client.mirror.0.46600.log.gz 2024-09-14T09:12:16.365 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71501.log 2024-09-14T09:12:16.374 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.47723.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.47723.log.gz 2024-09-14T09:12:16.375 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48783.log 2024-09-14T09:12:16.375 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.71501.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.71501.log.gz 2024-09-14T09:12:16.378 INFO:teuthology.orchestra.run.smithi164.stderr: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.47256.log.gz 2024-09-14T09:12:16.389 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49044.log 2024-09-14T09:12:16.390 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.48783.log: 79.6% -- replaced with /var/log/ceph/cluster2-client.mirror.48783.log.gz 2024-09-14T09:12:16.391 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68462.log 2024-09-14T09:12:16.394 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.49044.log: 95.2% -- replaced with /var/log/ceph/cluster1-client.mirror.49044.log.gz 2024-09-14T09:12:16.395 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55622.log 2024-09-14T09:12:16.395 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.68462.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.68462.log.gz 2024-09-14T09:12:16.396 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51916.log 2024-09-14T09:12:16.398 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.55622.log: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.55622.log.gz 2024-09-14T09:12:16.399 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73141.log 2024-09-14T09:12:16.400 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.51916.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.51916.log.gz 2024-09-14T09:12:16.400 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51330.log 2024-09-14T09:12:16.400 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.73141.log: 87.0% -- replaced with /var/log/ceph/cluster2-client.mirror.73141.log.gz 2024-09-14T09:12:16.401 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.37544.log 2024-09-14T09:12:16.403 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.51330.log: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.51330.log.gz 2024-09-14T09:12:16.404 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71666.log 2024-09-14T09:12:16.405 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.admin.37544.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.37544.log.gz 2024-09-14T09:12:16.405 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48386.log 2024-09-14T09:12:16.405 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.71666.log: 87.0% -- replaced with /var/log/ceph/cluster2-client.mirror.71666.log.gz 2024-09-14T09:12:16.406 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40992.log 2024-09-14T09:12:16.407 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.48386.log: 79.6% -- replaced with /var/log/ceph/cluster2-client.mirror.48386.log.gz 2024-09-14T09:12:16.407 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72664.log 2024-09-14T09:12:16.408 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.40992.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.40992.log.gz 2024-09-14T09:12:16.409 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56106.log 2024-09-14T09:12:16.410 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.72664.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.72664.log.gz 2024-09-14T09:12:16.410 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68646.log 2024-09-14T09:12:16.411 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.56106.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.56106.log.gz 2024-09-14T09:12:16.411 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66797.log 2024-09-14T09:12:16.412 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.68646.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.68646.log.gz 2024-09-14T09:12:16.413 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63699.log 2024-09-14T09:12:16.413 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.66797.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.66797.log.gz 2024-09-14T09:12:16.414 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52770.log 2024-09-14T09:12:16.415 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.63699.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.63699.log.gz 2024-09-14T09:12:16.415 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69749.log 2024-09-14T09:12:16.417 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.52770.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.52770.log.gz 2024-09-14T09:12:16.417 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42676.log 2024-09-14T09:12:16.418 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.69749.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.69749.log.gz 2024-09-14T09:12:16.418 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70581.log 2024-09-14T09:12:16.423 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.42676.log: 93.4% -- replaced with /var/log/ceph/cluster1-client.mirror.42676.log.gz 2024-09-14T09:12:16.424 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.39126.log 2024-09-14T09:12:16.425 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.70581.log: 87.0% -- replaced with /var/log/ceph/cluster2-client.mirror.70581.log.gz 2024-09-14T09:12:16.426 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46690.log 2024-09-14T09:12:16.426 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.39126.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.39126.log.gz 2024-09-14T09:12:16.427 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58536.log 2024-09-14T09:12:16.427 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.46690.log: 79.5% -- replaced with /var/log/ceph/cluster2-client.mirror.46690.log.gz 2024-09-14T09:12:16.428 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71687.log 2024-09-14T09:12:16.431 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.58536.log: 92.4% -- replaced with /var/log/ceph/cluster2-client.mirror.58536.log.gz 2024-09-14T09:12:16.431 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67269.log 2024-09-14T09:12:16.438 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.71687.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.71687.log.gz 2024-09-14T09:12:16.438 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.70140.log.gz 2024-09-14T09:12:16.438 INFO:teuthology.orchestra.run.smithi164.stderr: /var/log/ceph/cluster2-client.mirror.49955.log 2024-09-14T09:12:16.438 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65509.log 2024-09-14T09:12:16.438 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.67269.log: /var/log/ceph/cluster2-client.mirror.49955.log: 79.8% -- replaced with /var/log/ceph/cluster2-client.mirror.49955.log.gz 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.67269.log.gz 2024-09-14T09:12:16.438 INFO:teuthology.orchestra.run.smithi164.stderr: 2024-09-14T09:12:16.438 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67449.log 2024-09-14T09:12:16.438 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose/var/log/ceph/cluster2-client.mirror.65509.log: -- /var/log/ceph/cluster2-client.mirror.64154.log 2024-09-14T09:12:16.438 INFO:teuthology.orchestra.run.smithi164.stderr: 86.9% -- replaced with /var/log/ceph/cluster2-client.mirror.65509.log.gz 2024-09-14T09:12:16.438 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.67449.log: 86.9%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69661.log 2024-09-14T09:12:16.439 INFO:teuthology.orchestra.run.smithi164.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.67449.log.gz 2024-09-14T09:12:16.439 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.64154.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44113.log 2024-09-14T09:12:16.439 INFO:teuthology.orchestra.run.smithi164.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.64154.log.gz 2024-09-14T09:12:16.439 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.69661.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.62186.log 2024-09-14T09:12:16.439 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.44113.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.69661.log.gz 2024-09-14T09:12:16.439 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46452.log 2024-09-14T09:12:16.439 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.62186.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.62186.log.gz 2024-09-14T09:12:16.439 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67247.log 2024-09-14T09:12:16.440 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.46452.log: 87.3% -- replaced with /var/log/ceph/cluster2-client.mirror.46452.log.gz 2024-09-14T09:12:16.440 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71144.log 2024-09-14T09:12:16.440 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.67247.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.67247.log.gz 2024-09-14T09:12:16.441 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58399.log 2024-09-14T09:12:16.441 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.71144.log: 87.0% -- replaced with /var/log/ceph/cluster2-client.mirror.71144.log.gz 2024-09-14T09:12:16.451 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37433.log 2024-09-14T09:12:16.452 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.58399.log: 93.4% -- replaced with /var/log/ceph/cluster1-client.mirror.44113.log.gz 2024-09-14T09:12:16.452 INFO:teuthology.orchestra.run.smithi164.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.58399.log.gz 2024-09-14T09:12:16.452 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60326.log 2024-09-14T09:12:16.452 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.37433.log: 2.3%gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.36167.log 2024-09-14T09:12:16.452 INFO:teuthology.orchestra.run.smithi164.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.37433.log.gz 2024-09-14T09:12:16.452 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.60326.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70660.log 2024-09-14T09:12:16.452 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.admin.36167.log: 87.3% -- replaced with /var/log/ceph/cluster2-client.mirror.60326.log.gz 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.36167.log.gz 2024-09-14T09:12:16.452 INFO:teuthology.orchestra.run.smithi164.stderr: 2024-09-14T09:12:16.452 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73705.log 2024-09-14T09:12:16.452 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.70660.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69168.log 2024-09-14T09:12:16.453 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.73705.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.70660.log.gz 2024-09-14T09:12:16.453 INFO:teuthology.orchestra.run.smithi164.stderr: 87.0% -- replaced with /var/log/ceph/cluster2-client.mirror.73705.log.gz 2024-09-14T09:12:16.453 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47305.log 2024-09-14T09:12:16.453 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.69168.log: 79.8%gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66549.log 2024-09-14T09:12:16.453 INFO:teuthology.orchestra.run.smithi164.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.69168.log.gz 2024-09-14T09:12:16.453 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56765.log 2024-09-14T09:12:16.453 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.66549.log: /var/log/ceph/cluster2-client.mirror.47305.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.66549.log.gz 2024-09-14T09:12:16.453 INFO:teuthology.orchestra.run.smithi164.stderr: 88.7%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69292.log 2024-09-14T09:12:16.453 INFO:teuthology.orchestra.run.smithi164.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.47305.log.gz 2024-09-14T09:12:16.453 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.56765.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53623.log 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.56765.log.gz 2024-09-14T09:12:16.453 INFO:teuthology.orchestra.run.smithi164.stderr: 2024-09-14T09:12:16.453 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.39689.log 2024-09-14T09:12:16.453 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.69292.log: /var/log/ceph/cluster1-client.mirror.53623.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.69292.log.gz 2024-09-14T09:12:16.454 INFO:teuthology.orchestra.run.smithi164.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.53623.log.gz 2024-09-14T09:12:16.454 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.62737.log 2024-09-14T09:12:16.454 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.39689.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57628.log 2024-09-14T09:12:16.454 INFO:teuthology.orchestra.run.smithi164.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.39689.log.gz 2024-09-14T09:12:16.454 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.62737.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.62600.log 2024-09-14T09:12:16.475 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.57628.log: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.62737.log.gz 2024-09-14T09:12:16.475 INFO:teuthology.orchestra.run.smithi164.stderr: 92.2%gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66887.log 2024-09-14T09:12:16.476 INFO:teuthology.orchestra.run.smithi164.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.57628.log.gz 2024-09-14T09:12:16.476 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55332.log 2024-09-14T09:12:16.476 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.62600.log: /var/log/ceph/cluster1-client.mirror.66887.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.62600.log.gz 2024-09-14T09:12:16.476 INFO:teuthology.orchestra.run.smithi164.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.66887.log.gz 2024-09-14T09:12:16.476 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70832.log 2024-09-14T09:12:16.476 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45712.log 2024-09-14T09:12:16.476 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.55332.log: /var/log/ceph/cluster2-client.mirror.70832.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.70832.log.gz 2024-09-14T09:12:16.476 INFO:teuthology.orchestra.run.smithi164.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.55332.log.gz 2024-09-14T09:12:16.476 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53599.log 2024-09-14T09:12:16.476 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.45712.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41535.log 2024-09-14T09:12:16.476 INFO:teuthology.orchestra.run.smithi164.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.45712.log.gz 2024-09-14T09:12:16.476 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.53599.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42698.log 2024-09-14T09:12:16.476 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.41535.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.53599.log.gz 2024-09-14T09:12:16.477 INFO:teuthology.orchestra.run.smithi164.stderr: 88.5%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37785.log 2024-09-14T09:12:16.477 INFO:teuthology.orchestra.run.smithi164.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.41535.log.gz 2024-09-14T09:12:16.477 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65116.log 2024-09-14T09:12:16.477 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.42698.log: /var/log/ceph/cluster2-client.mirror.37785.log: 79.6% -- replaced with /var/log/ceph/cluster2-client.mirror.37785.log.gz 2024-09-14T09:12:16.477 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72684.log 2024-09-14T09:12:16.477 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.65116.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.42698.log.gz 2024-09-14T09:12:16.477 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.74654.log 2024-09-14T09:12:16.477 INFO:teuthology.orchestra.run.smithi164.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.65116.log.gz 2024-09-14T09:12:16.477 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.72684.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50370.log 2024-09-14T09:12:16.477 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.admin.74654.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.74654.log.gz 2024-09-14T09:12:16.477 INFO:teuthology.orchestra.run.smithi164.stderr: 86.9% -- replaced with /var/log/ceph/cluster2-client.mirror.72684.log.gz 2024-09-14T09:12:16.477 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65777.log 2024-09-14T09:12:16.478 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.50370.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64237.log 2024-09-14T09:12:16.478 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.65777.log: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.50370.log.gz 2024-09-14T09:12:16.478 INFO:teuthology.orchestra.run.smithi164.stderr: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.65777.log.gz 2024-09-14T09:12:16.478 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45225.log 2024-09-14T09:12:16.478 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.64237.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59594.log 2024-09-14T09:12:16.478 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.45225.log: 79.6% -- replaced with /var/log/ceph/cluster2-client.mirror.45225.log.gz 2024-09-14T09:12:16.478 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.40472.log 2024-09-14T09:12:16.478 INFO:teuthology.orchestra.run.smithi164.stderr: 90.1% -- replaced with /var/log/ceph/cluster2-client.mirror.64237.log.gz 2024-09-14T09:12:16.478 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.59594.log: gzip 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.59594.log.gz 2024-09-14T09:12:16.478 INFO:teuthology.orchestra.run.smithi164.stderr: -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70355.log 2024-09-14T09:12:16.478 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44351.log 2024-09-14T09:12:16.478 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.admin.40472.log: /var/log/ceph/cluster1-client.mirror.70355.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.40472.log.gz 2024-09-14T09:12:16.478 INFO:teuthology.orchestra.run.smithi164.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.70355.log.gz 2024-09-14T09:12:16.478 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60398.log 2024-09-14T09:12:16.478 INFO:teuthology.orchestra.run.smithi164.stderr: 93.7%/var/log/ceph/cluster2-client.mirror.44351.log: -- replaced with /var/log/ceph/cluster2-client.mirror.40839.log.gz 2024-09-14T09:12:16.479 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47564.log 2024-09-14T09:12:16.479 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67472.log 2024-09-14T09:12:16.479 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.60398.log: /var/log/ceph/cluster2-client.mirror.47564.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.60398.log.gz 2024-09-14T09:12:16.479 INFO:teuthology.orchestra.run.smithi164.stderr: 87.4%gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37450.log 2024-09-14T09:12:16.479 INFO:teuthology.orchestra.run.smithi164.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.47564.log.gz 2024-09-14T09:12:16.479 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57129.log 2024-09-14T09:12:16.479 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.67472.log: /var/log/ceph/cluster1-client.mirror.37450.log: 56.7% -- replaced with /var/log/ceph/cluster1-client.mirror.37450.log.gz 2024-09-14T09:12:16.479 INFO:teuthology.orchestra.run.smithi164.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.67472.log.gz 2024-09-14T09:12:16.479 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47426.log 2024-09-14T09:12:16.479 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.57129.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41136.log 2024-09-14T09:12:16.479 INFO:teuthology.orchestra.run.smithi164.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.57129.log.gz 2024-09-14T09:12:16.479 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.47426.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56437.log 2024-09-14T09:12:16.479 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.41136.log: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.41136.log.gz 2024-09-14T09:12:16.479 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.44351.log.gz 2024-09-14T09:12:16.480 INFO:teuthology.orchestra.run.smithi164.stderr: --verbose -- /var/log/ceph/cluster2-client.mirror.56164.log 2024-09-14T09:12:16.480 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.56437.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.1.48297.log 2024-09-14T09:12:16.480 INFO:teuthology.orchestra.run.smithi164.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.56437.log.gz 2024-09-14T09:12:16.480 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.56164.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.73816.log 2024-09-14T09:12:16.480 INFO:teuthology.orchestra.run.smithi164.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.56164.log.gz 2024-09-14T09:12:16.480 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.1.48297.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45929.log 2024-09-14T09:12:16.480 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.73816.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.mirror.73816.log.gz 2024-09-14T09:12:16.480 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61046.log 2024-09-14T09:12:16.480 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.45929.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.45929.log.gz 2024-09-14T09:12:16.480 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65157.log 2024-09-14T09:12:16.482 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.61046.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.61046.log.gz 2024-09-14T09:12:16.482 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57346.log 2024-09-14T09:12:16.482 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.65157.log: 90.1% -- replaced with /var/log/ceph/cluster2-client.mirror.65157.log.gz 2024-09-14T09:12:16.482 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47347.log 2024-09-14T09:12:16.482 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.57346.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.57346.log.gz 2024-09-14T09:12:16.482 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.35916.log 2024-09-14T09:12:16.494 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.47347.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.3.37055.log 2024-09-14T09:12:16.495 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.admin.35916.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.35916.log.gz 2024-09-14T09:12:16.496 INFO:teuthology.orchestra.run.smithi164.stderr: 94.0% -- replaced with /var/log/ceph/cluster2-client.mirror.1.48297.log.gz 2024-09-14T09:12:16.497 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.71034.log 2024-09-14T09:12:16.521 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.3.37055.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72163.log 2024-09-14T09:12:16.522 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.2.71034.log: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.2.71034.log.gz 2024-09-14T09:12:16.527 INFO:teuthology.orchestra.run.smithi164.stderr: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.38399.log.gz 2024-09-14T09:12:16.530 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66067.log 2024-09-14T09:12:16.530 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.72163.log: 87.0% -- replaced with /var/log/ceph/cluster2-client.mirror.72163.log.gz 2024-09-14T09:12:16.530 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49707.log 2024-09-14T09:12:16.530 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.66067.log: 93.6% -- replaced with /var/log/ceph/cluster2-client.mirror.3.37055.log.gz 2024-09-14T09:12:16.530 INFO:teuthology.orchestra.run.smithi164.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.66067.log.gz 2024-09-14T09:12:16.530 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.2.37027.log 2024-09-14T09:12:16.530 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.49707.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.35657.log 2024-09-14T09:12:16.534 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.2.37027.log: 95.6% -- replaced with /var/log/ceph/cluster1-client.mirror.49707.log.gz 2024-09-14T09:12:16.534 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51289.log 2024-09-14T09:12:16.534 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.admin.35657.log: 80.3% -- replaced with /var/log/ceph/cluster2-client.admin.35657.log.gz 2024-09-14T09:12:16.534 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70205.log 2024-09-14T09:12:16.535 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.51289.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.51289.log.gz 2024-09-14T09:12:16.536 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61437.log 2024-09-14T09:12:16.536 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster2-client.mirror.70205.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.70205.log.gz 2024-09-14T09:12:16.537 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cluster1-client.mirror.61437.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.61437.log.gz 2024-09-14T09:12:16.590 INFO:teuthology.orchestra.run.smithi164.stderr: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.48065.log.gz 2024-09-14T09:12:16.616 INFO:teuthology.orchestra.run.smithi164.stderr: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.47347.log.gz 2024-09-14T09:12:16.657 INFO:teuthology.orchestra.run.smithi164.stderr: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.2.37027.log.gz 2024-09-14T09:12:16.777 INFO:teuthology.orchestra.run.smithi164.stderr: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.47426.log.gz 2024-09-14T09:12:20.198 INFO:teuthology.orchestra.run.smithi135.stderr: 94.4% -- replaced with /var/log/ceph/cluster1-osd.1.log.gz 2024-09-14T09:12:20.753 INFO:teuthology.orchestra.run.smithi135.stderr: 94.5% -- replaced with /var/log/ceph/cluster1-osd.2.log.gz 2024-09-14T09:12:20.793 INFO:teuthology.orchestra.run.smithi135.stderr: 94.4% -- replaced with /var/log/ceph/cluster1-osd.0.log.gz 2024-09-14T09:12:20.795 INFO:teuthology.orchestra.run.smithi135.stderr: 2024-09-14T09:12:20.795 INFO:teuthology.orchestra.run.smithi135.stderr:real 0m8.307s 2024-09-14T09:12:20.796 INFO:teuthology.orchestra.run.smithi135.stderr:user 0m24.800s 2024-09-14T09:12:20.796 INFO:teuthology.orchestra.run.smithi135.stderr:sys 0m1.002s 2024-09-14T09:12:55.823 INFO:teuthology.orchestra.run.smithi164.stderr: 93.6% -- replaced with /var/log/ceph/cluster2-osd.2.log.gz 2024-09-14T09:13:01.168 INFO:teuthology.orchestra.run.smithi164.stderr: 93.6% -- replaced with /var/log/ceph/cluster2-osd.1.log.gz 2024-09-14T09:13:11.561 INFO:teuthology.orchestra.run.smithi164.stderr: 93.6% -- replaced with /var/log/ceph/cluster2-osd.0.log.gz 2024-09-14T09:13:11.563 INFO:teuthology.orchestra.run.smithi164.stderr: 2024-09-14T09:13:11.563 INFO:teuthology.orchestra.run.smithi164.stderr:real 0m59.036s 2024-09-14T09:13:11.563 INFO:teuthology.orchestra.run.smithi164.stderr:user 2m43.358s 2024-09-14T09:13:11.563 INFO:teuthology.orchestra.run.smithi164.stderr:sys 0m7.020s 2024-09-14T09:13:11.564 INFO:tasks.ceph:Archiving logs... 2024-09-14T09:13:11.564 DEBUG:teuthology.misc:Transferring archived files from smithi135:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-09-10_20:16:17-rbd-main-distro-default-smithi/7900492/remote/smithi135/log 2024-09-14T09:13:11.565 DEBUG:teuthology.orchestra.run.smithi135:> sudo tar c -f - -C /var/log/ceph -- . 2024-09-14T09:13:14.950 DEBUG:teuthology.misc:Transferring archived files from smithi164:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-09-10_20:16:17-rbd-main-distro-default-smithi/7900492/remote/smithi164/log 2024-09-14T09:13:14.951 DEBUG:teuthology.orchestra.run.smithi164:> sudo tar c -f - -C /var/log/ceph -- . 2024-09-14T09:13:39.231 DEBUG:teuthology.run_tasks:Unwinding manager install 2024-09-14T09:13:39.252 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer... 2024-09-14T09:13:39.253 DEBUG:teuthology.orchestra.run.smithi135:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-09-14T09:13:39.255 DEBUG:teuthology.orchestra.run.smithi164:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-09-14T09:13:39.334 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-09-14T09:13:39.349 INFO:teuthology.task.clock:Checking final clock skew... 2024-09-14T09:13:39.349 DEBUG:teuthology.orchestra.run.smithi135:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-14T09:13:39.352 DEBUG:teuthology.orchestra.run.smithi164:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-14T09:13:39.366 INFO:teuthology.orchestra.run.smithi135.stderr:bash: line 1: ntpq: command not found 2024-09-14T09:13:39.370 INFO:teuthology.orchestra.run.smithi164.stderr:bash: line 1: ntpq: command not found 2024-09-14T09:13:39.371 INFO:teuthology.orchestra.run.smithi135.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-14T09:13:39.371 INFO:teuthology.orchestra.run.smithi135.stdout:=============================================================================== 2024-09-14T09:13:39.372 INFO:teuthology.orchestra.run.smithi135.stdout:^* hv01.front.sepia.ceph.com 3 8 377 32 +1076us[+1102us] +/- 40ms 2024-09-14T09:13:39.372 INFO:teuthology.orchestra.run.smithi135.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 29 -1372us[-1372us] +/- 56ms 2024-09-14T09:13:39.372 INFO:teuthology.orchestra.run.smithi135.stdout:^- hv03.front.sepia.ceph.com 3 6 377 26 -7935us[-7935us] +/- 95ms 2024-09-14T09:13:39.372 INFO:teuthology.orchestra.run.smithi135.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-09-14T09:13:39.375 INFO:teuthology.orchestra.run.smithi164.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-14T09:13:39.375 INFO:teuthology.orchestra.run.smithi164.stdout:=============================================================================== 2024-09-14T09:13:39.375 INFO:teuthology.orchestra.run.smithi164.stdout:^* hv01.front.sepia.ceph.com 3 6 377 31 +1028us[+1012us] +/- 40ms 2024-09-14T09:13:39.375 INFO:teuthology.orchestra.run.smithi164.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 25 -1389us[-1389us] +/- 56ms 2024-09-14T09:13:39.375 INFO:teuthology.orchestra.run.smithi164.stdout:^- hv03.front.sepia.ceph.com 3 6 377 27 -7959us[-7959us] +/- 95ms 2024-09-14T09:13:39.375 INFO:teuthology.orchestra.run.smithi164.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-09-14T09:13:39.376 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-09-14T09:13:39.390 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-09-14T09:13:39.390 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-09-14T09:13:39.407 DEBUG:teuthology.orchestra.run.smithi135:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-09-14T09:13:39.479 DEBUG:teuthology.orchestra.run.smithi164:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-09-14T09:13:39.550 DEBUG:teuthology.orchestra.run.smithi135:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 2024-09-14T09:13:39.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T09:13:39.580 DEBUG:teuthology.orchestra.run.smithi164:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 2024-09-14T09:13:39.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T09:13:39.611 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-09-14T09:13:39.625 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-09-14T09:13:39.642 INFO:teuthology.task.internal:Duration was 2846.636310 seconds 2024-09-14T09:13:39.643 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-09-14T09:13:39.659 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-09-14T09:13:39.659 DEBUG:teuthology.orchestra.run.smithi135:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-09-14T09:13:39.662 DEBUG:teuthology.orchestra.run.smithi164:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-09-14T09:13:39.699 INFO:teuthology.orchestra.run.smithi135.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-14T09:13:39.704 INFO:teuthology.orchestra.run.smithi164.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-14T09:13:40.003 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-09-14T09:13:40.003 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi135.front.sepia.ceph.com 2024-09-14T09:13:40.004 DEBUG:teuthology.orchestra.run.smithi135:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-09-14T09:13:40.024 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi164.front.sepia.ceph.com 2024-09-14T09:13:40.024 DEBUG:teuthology.orchestra.run.smithi164:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-09-14T09:13:40.064 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-09-14T09:13:40.064 DEBUG:teuthology.orchestra.run.smithi135:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-14T09:13:40.067 DEBUG:teuthology.orchestra.run.smithi164:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-14T09:13:40.132 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-09-14T09:13:40.132 DEBUG:teuthology.orchestra.run.smithi135:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-09-14T09:13:40.135 DEBUG:teuthology.orchestra.run.smithi164:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-09-14T09:13:40.275 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-09-14T09:13:40.288 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-09-14T09:13:40.288 DEBUG:teuthology.orchestra.run.smithi135:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-09-14T09:13:40.313 DEBUG:teuthology.orchestra.run.smithi164:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-09-14T09:13:40.338 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-09-14T09:13:40.353 DEBUG:teuthology.orchestra.run.smithi135:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-09-14T09:13:40.356 DEBUG:teuthology.orchestra.run.smithi164:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-09-14T09:13:40.377 INFO:teuthology.orchestra.run.smithi135.stdout:kernel.core_pattern = core 2024-09-14T09:13:40.401 INFO:teuthology.orchestra.run.smithi164.stdout:kernel.core_pattern = core 2024-09-14T09:13:40.416 DEBUG:teuthology.orchestra.run.smithi135:> test -e /home/ubuntu/cephtest/archive/coredump 2024-09-14T09:13:40.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T09:13:40.471 DEBUG:teuthology.orchestra.run.smithi164:> test -e /home/ubuntu/cephtest/archive/coredump 2024-09-14T09:13:40.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T09:13:40.486 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-09-14T09:13:40.499 INFO:teuthology.task.internal:Transferring archived files... 2024-09-14T09:13:40.500 DEBUG:teuthology.misc:Transferring archived files from smithi135:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-09-10_20:16:17-rbd-main-distro-default-smithi/7900492/remote/smithi135 2024-09-14T09:13:40.500 DEBUG:teuthology.orchestra.run.smithi135:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-09-14T09:13:40.547 DEBUG:teuthology.misc:Transferring archived files from smithi164:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-09-10_20:16:17-rbd-main-distro-default-smithi/7900492/remote/smithi164 2024-09-14T09:13:40.547 DEBUG:teuthology.orchestra.run.smithi164:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-09-14T09:13:40.580 INFO:teuthology.task.internal:Removing archive directory... 2024-09-14T09:13:40.580 DEBUG:teuthology.orchestra.run.smithi135:> rm -rf -- /home/ubuntu/cephtest/archive 2024-09-14T09:13:40.584 DEBUG:teuthology.orchestra.run.smithi164:> rm -rf -- /home/ubuntu/cephtest/archive 2024-09-14T09:13:40.630 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-09-14T09:13:40.645 INFO:teuthology.task.internal:Not uploading archives. 2024-09-14T09:13:40.645 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-09-14T09:13:40.658 INFO:teuthology.task.internal:Tidying up after the test... 2024-09-14T09:13:40.658 DEBUG:teuthology.orchestra.run.smithi135:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-09-14T09:13:40.660 DEBUG:teuthology.orchestra.run.smithi164:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-09-14T09:13:40.673 INFO:teuthology.orchestra.run.smithi135.stdout: 262169 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 14 09:13 /home/ubuntu/cephtest 2024-09-14T09:13:40.686 INFO:teuthology.orchestra.run.smithi164.stdout: 262169 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 14 09:13 /home/ubuntu/cephtest 2024-09-14T09:13:40.687 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-09-14T09:13:40.701 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-09-14T09:13:40.758 INFO:teuthology.run:Summary data: description: rbd/mirror-thrash/{base/install clients/mirror cluster/{2-node openstack} conf/{disable-pool-app} msgr-failures/few objectstore/bluestore-comp-zlib policy/simple rbd-mirror/four-per-cluster supported-random-distro$/{centos_latest} workloads/rbd-mirror-snapshot-stress-workunit-minimum} duration: 2846.6363101005554 flavor: default owner: scheduled_teuthology@teuthology success: true 2024-09-14T09:13:40.758 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-14T09:13:40.836 INFO:teuthology.run:pass